From 3ca6c26ec1af2c713186f9cffbd27790e25b040e Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Fri, 25 Sep 2026 19:13:11 +0530 Subject: [PATCH 1/7] perf(index): throttle index saves and report each leg's save state Every observation, compression and memory save scheduled a full index save 5 s later, and each save re-serializes the whole BM25 and vector index and rewrites every shard. On a busy store that is a full index rewrite almost continuously. Saves now run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS (default 10 minutes), measured from the start of the previous save. Explicit saves (deletes, shutdown) still run immediately. Only one save runs at a time; saves requested while one is running collapse into a single follow-up save. The BM25 and vector saves now fail independently. Before, a BM25 failure skipped the vector save in the same run and both were reported as one warning. Each leg records when it last saved, its last error, since when it has unsaved changes, and its serialized size. /agentmemory/status reports a failing leg as an error, unsaved changes older than twice the interval as a warning, and a vector index above 80% of the Node.js string limit (536,870,888 characters) as a warning, since past that limit the index can no longer be saved or loaded. The status page shows each leg's save state and the vector index size. --- .env.example | 1 + plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/config.ts | 10 + src/functions/search.ts | 16 +- src/functions/status.ts | 67 ++++ src/state/index-persistence.ts | 172 ++++++++-- src/triggers/api.ts | 3 +- test/index-persistence-throttle.test.ts | 302 ++++++++++++++++++ test/index-persistence.test.ts | 6 +- 9 files changed, 539 insertions(+), 41 deletions(-) create mode 100644 test/index-persistence-throttle.test.ts diff --git a/.env.example b/.env.example index 9219a338a1..4b9deb0b60 100644 --- a/.env.example +++ b/.env.example @@ -129,6 +129,7 @@ # AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). +# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index d152a7c62b..5fe071567b 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 43 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -27,6 +27,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` - `AGENTMEMORY_IMAGE_STORE_MAX_BYTES` +- `AGENTMEMORY_INDEX_SAVE_INTERVAL_MS` - `AGENTMEMORY_INJECT_CONTEXT` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` diff --git a/src/config.ts b/src/config.ts index a6c62ce55e..ac82bd445f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -467,6 +467,16 @@ export function getConsolidationCooldownMs(): number { return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS; } +export const INDEX_SAVE_INTERVAL_DEFAULT_MS = 600_000; + +export function getIndexSaveIntervalMs(): number { + const raw = safeParseInt( + getMergedEnv()["AGENTMEMORY_INDEX_SAVE_INTERVAL_MS"], + INDEX_SAVE_INTERVAL_DEFAULT_MS, + ); + return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/search.ts b/src/functions/search.ts index 2b8858bdc5..8abce07f45 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -1,4 +1,5 @@ import type { IIIClient } from 'iii-sdk' +import type { IndexPersistenceStatus } from "../state/index-persistence.js"; import type { CompactSearchResult, CompressedObservation, Memory, SearchResult, Session } from '../types.js' import { KV } from '../state/schema.js' import { StateKV } from '../state/kv.js' @@ -75,17 +76,22 @@ export function vectorIndexRemove(id: string): void { // Wired by src/index.ts after IndexPersistence is constructed; no-op // until then so unit tests that exercise the delete paths in // isolation don't need to wire persistence. -let indexPersistence: { +type IndexPersistenceHook = { scheduleSave: () => void; save: () => Promise; -} | null = null; + status?: () => IndexPersistenceStatus; +}; -export function setIndexPersistence( - p: { scheduleSave: () => void; save: () => Promise } | null, -): void { +let indexPersistence: IndexPersistenceHook | null = null; + +export function setIndexPersistence(p: IndexPersistenceHook | null): void { indexPersistence = p; } +export function getIndexPersistenceStatus(): IndexPersistenceStatus | null { + return indexPersistence?.status?.() ?? null; +} + export function scheduleIndexSave(): void { indexPersistence?.scheduleSave(); } diff --git a/src/functions/status.ts b/src/functions/status.ts index 552fe6c07a..0b3ab5310a 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -1,3 +1,5 @@ +import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js"; + export type StatusLevel = "ok" | "info" | "warn" | "error"; export interface StatusProblem { @@ -58,6 +60,7 @@ export interface StatusInputs { }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; + indexPersistence?: IndexPersistenceStatus | null; } export interface StatusReport { @@ -72,6 +75,7 @@ export interface StatusReport { health: StatusInputs["health"]; provider: { llm: string; embeddings: string; circuitBreaker: StatusInputs["circuitBreaker"] }; index: StatusInputs["index"]; + indexPersistence: IndexPersistenceStatus | null; graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null; functions: Array; flags: StatusFlag[]; @@ -81,6 +85,7 @@ export interface StatusReport { const FAILURE_RATE_THRESHOLD = 0.2; const FAILURE_MIN_CALLS = 5; const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60; +const VECTOR_STRING_WARN_RATIO = 0.8; const LEVEL_RANK: Record = { ok: 0, info: 1, warn: 2, error: 3 }; const FUNCTION_FIXES: Record = { @@ -168,6 +173,43 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } + const persistence = input.indexPersistence ?? null; + if (persistence) { + const legs: Array<[string, IndexLegStatus | null]> = [ + ["BM25", persistence.bm25], + ["vector", persistence.vector], + ]; + for (const [label, leg] of legs) { + if (!leg) continue; + if (leg.lastError) { + problems.push({ + level: "error", + code: "index-save-failing", + message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`, + fix: "Check the server log for the failing state write. The next save retries automatically.", + }); + } + const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined); + if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { + problems.push({ + level: "warn", + code: "index-save-stale", + message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, + fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", + }); + } + } + const vectorChars = persistence.vector?.serializedChars ?? null; + if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) { + problems.push({ + level: "warn", + code: "vector-index-near-string-limit", + message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`, + fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.", + }); + } + } + let graph: StatusReport["graph"] = null; if (input.graph) { const ageSeconds = secondsBetween(input.now, input.graph.updatedAt); @@ -218,6 +260,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport { circuitBreaker: input.circuitBreaker, }, index: input.index, + indexPersistence: persistence, graph, functions, flags: input.flags, @@ -248,6 +291,29 @@ function row(label: string, value: string): string { return `${escapeHtml(label)}${value}`; } +function legSummary(report: StatusReport, leg: IndexLegStatus): string { + if (leg.lastError) return `failing: ${leg.lastError}`; + const saved = leg.lastSavedAt ? `saved ${formatDuration(secondsBetween(new Date(report.checkedAt), leg.lastSavedAt))} ago` : "not saved since start"; + return leg.dirtySince ? `${saved}, unsaved changes pending` : saved; +} + +function indexPersistenceRows(report: StatusReport): string { + const persistence = report.indexPersistence; + if (!persistence) return ""; + let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25))); + if (persistence.vector) { + rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector))); + const chars = persistence.vector.serializedChars; + if (chars !== null) { + rows += row( + "Vector index size", + escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`), + ); + } + } + return rows; +} + export function renderStatusHtml(report: StatusReport, styleNonce: string): string { const problems = report.problems.length ? report.problems @@ -330,6 +396,7 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off") ${row("Observations indexed", escapeHtml(idx.observationsIndexed))} ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))} ${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))} +${indexPersistenceRows(report)}

Knowledge graph

${graph diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 19b05cb598..a77d66ed30 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -4,8 +4,9 @@ import type { StateKV } from "./kv.js"; import { KV, generateId } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; +import { getIndexSaveIntervalMs } from "../config.js"; -const DEBOUNCE_MS = 5000; +export const V8_MAX_STRING_CHARS = 536_870_888; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; const BM25_KEY = "data"; @@ -41,8 +42,31 @@ type IndexShardManifest = { type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; + saveIntervalMs?: number; + now?: () => number; }; +export type IndexLeg = "bm25" | "vector"; + +export interface IndexLegStatus { + lastSavedAt: string | null; + lastError: string | null; + lastErrorAt: string | null; + dirtySince: string | null; + serializedChars: number | null; +} + +export interface IndexPersistenceStatus { + saveIntervalMs: number; + saving: boolean; + bm25: IndexLegStatus; + vector: IndexLegStatus | null; +} + +function emptyLegStatus(): IndexLegStatus { + return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null }; +} + function shardChars(options: IndexPersistenceOptions): number { const configured = options.shardChars; if (typeof configured !== "number" || !Number.isFinite(configured)) { @@ -81,39 +105,72 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private running: Promise | null = null; + private queued: Promise | null = null; + private stopped = false; + private lastSaveAt: number; + private dirtyEpoch = 0; + private markedDuringRunAt: number | null = null; + private legs: Record = { + bm25: emptyLegStatus(), + vector: emptyLegStatus(), + }; + private readonly saveIntervalMs: number; + private readonly now: () => number; constructor( private kv: StateKV, private bm25: SearchIndex, private vector: VectorIndex | null, private options: IndexPersistenceOptions = {}, - ) {} + ) { + this.now = options.now ?? Date.now; + const interval = options.saveIntervalMs; + this.saveIntervalMs = + typeof interval === "number" && Number.isFinite(interval) && interval > 0 + ? interval + : getIndexSaveIntervalMs(); + this.lastSaveAt = this.now(); + } scheduleSave(): void { - if (this.timer) clearTimeout(this.timer); - // setTimeout discards the returned promise, so any rejection inside - // save() would surface as unhandledRejection and crash the process - // under sustained iii-engine write timeouts (issue #204). Funnel - // rejections through logFailure() instead. + if (this.stopped) return; + const now = this.now(); + this.dirtyEpoch++; + if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now; + for (const leg of this.activeLegs()) { + if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString(); + } + if (this.timer) return; + const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now); this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); - }, DEBOUNCE_MS); + this.timer = null; + this.save().catch((err) => this.logFailure("bm25", err)); + }, delay); } - async save(): Promise { - if (this.timer) { - clearTimeout(this.timer); - this.timer = null; - } - try { - await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { - await this.saveVectorIndex(this.vector.serialize()); - } - } catch (err) { - this.logFailure(err); + save(): Promise { + this.clearTimer(); + if (this.queued) return this.queued; + if (this.running) { + const queued = this.running.then(() => { + this.queued = null; + return this.startRun(); + }); + this.queued = queued; + return queued; } + return this.startRun(); + } + + status(): IndexPersistenceStatus { + return { + saveIntervalMs: this.saveIntervalMs, + saving: this.running !== null, + bm25: { ...this.legs.bm25 }, + vector: this.vector ? { ...this.legs.vector } : null, + }; } async load(): Promise<{ @@ -137,27 +194,80 @@ export class IndexPersistence { } stop(): void { + this.stopped = true; + this.clearTimer(); + } + + private clearTimer(): void { if (this.timer) { clearTimeout(this.timer); this.timer = null; } } - private logFailure(err: unknown): void { - const now = Date.now(); - // Throttle: persistence failures under load arrive in bursts - // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + private activeLegs(): IndexLeg[] { + return this.vector ? ["bm25", "vector"] : ["bm25"]; + } + + private startRun(): Promise { + const run = this.runSave().finally(() => { + if (this.running === run) this.running = null; + }); + this.running = run; + return run; + } + + private async runSave(): Promise { + const epoch = this.dirtyEpoch; + this.markedDuringRunAt = null; + this.lastSaveAt = this.now(); + await this.saveLeg("bm25", epoch, async () => { + const serialized = this.bm25.serialize(); + this.legs.bm25.serializedChars = serialized.length; + await this.saveBm25Index(serialized); + }); + const vector = this.vector; + if (vector) { + await this.saveLeg("vector", epoch, async () => { + const serialized = vector.serialize(); + this.legs.vector.serializedChars = serialized.length; + await this.saveVectorIndex(serialized); + }); + } + } + + private async saveLeg(leg: IndexLeg, epoch: number, run: () => Promise): Promise { + const status = this.legs[leg]; + try { + await run(); + status.lastSavedAt = new Date(this.now()).toISOString(); + status.lastError = null; + status.lastErrorAt = null; + if (this.dirtyEpoch === epoch) { + status.dirtySince = null; + } else if (this.markedDuringRunAt !== null) { + status.dirtySince = new Date(this.markedDuringRunAt).toISOString(); + } + } catch (err) { + status.lastError = errorMessage(err); + status.lastErrorAt = new Date(this.now()).toISOString(); + if (status.dirtySince === null) status.dirtySince = status.lastErrorAt; + this.logFailure(leg, err); + } + } + + private logFailure(leg: IndexLeg, err: unknown): void { + const now = this.now(); + if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(leg, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, { code, message, hint: code === "TIMEOUT" - ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next debounce flush" + ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save" : undefined, }); } diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d8a447f2e0..2fc2f3285a 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -12,7 +12,7 @@ import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js"; -import { findUnindexedObservations, getSearchIndex, getVectorIndex } from "../functions/search.js"; +import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js"; import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; @@ -373,6 +373,7 @@ export function registerApiTriggers( }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), + indexPersistence: getIndexPersistenceStatus(), }); const accept = req.headers?.["accept"] ?? req.headers?.["Accept"]; const format = req.query_params?.["format"]; diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts new file mode 100644 index 0000000000..8a013fa662 --- /dev/null +++ b/test/index-persistence-throttle.test.ts @@ -0,0 +1,302 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js"; +import { SearchIndex } from "../src/state/search-index.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js"; +import type { CompressedObservation } from "../src/types.js"; + +const BM25_SCOPE = "mem:index:bm25"; + +function mockKV() { + const store = new Map>(); + const sets: Array<{ scope: string; key: string }> = []; + return { + sets, + store, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + sets.push({ scope, key }); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function obs(id: string, title: string): CompressedObservation { + return { + id, + sessionId: "ses_1", + timestamp: new Date().toISOString(), + type: "file_edit", + title, + facts: [], + narrative: `${title} narrative`, + concepts: [], + files: [], + importance: 5, + }; +} + +function manifestSaves(kv: ReturnType, key: string): number { + return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length; +} + +describe("IndexPersistence save throttling", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("saves at most once per interval however often changes are scheduled", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(59_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + + await vi.advanceTimersByTimeAsync(1_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(2); + }); + + it("an explicit save runs immediately and cancels the pending timer", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + await vi.advanceTimersByTimeAsync(120_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + }); + + it("never runs two saves at once and coalesces requests made during a save into one", async () => { + let release: () => void = () => undefined; + let inFlight = 0; + let maxInFlight = 0; + let gate: Promise | null = new Promise((resolve) => { + release = resolve; + }); + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + if (gate) await gate; + const result = await kv.set(scope, key, data); + inFlight--; + return result; + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const first = persistence.save(); + const second = persistence.save(); + const third = persistence.save(); + expect(second).toBe(third); + expect(persistence.status().saving).toBe(true); + + gate = null; + release(); + await Promise.all([first, second, third]); + + expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(maxInFlight).toBe(1); + expect(persistence.status().saving).toBe(false); + }); + + it("a failing BM25 leg does not stop the vector leg from saving", async () => { + const failingKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed"); + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + + const status = persistence.status(); + expect(status.bm25.lastError).toBe("bm25 manifest write failed"); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.lastError).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.serializedChars).toBeGreaterThan(0); + + const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + expect(loaded.bm25).toBeNull(); + expect(loaded.vector?.size).toBe(1); + }); + + it("keeps a leg dirty when a change arrives while it is being saved", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let held = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (!held) { + held = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + expect(persistence.status().bm25.lastSavedAt).not.toBeNull(); + expect(persistence.status().bm25.dirtySince).not.toBeNull(); + + await persistence.save(); + expect(persistence.status().bm25.dirtySince).toBeNull(); + }); + + it("stop prevents later scheduled saves", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 }); + persistence.stop(); + persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(10_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + }); +}); + +function statusInputs(overrides: Partial = {}): StatusInputs { + return { + now: new Date("2026-09-25T12:00:00.000Z"), + version: "0.9.29", + engineVersion: "0.22.1", + uptimeSeconds: 100, + ports: { rest: 3111, streams: 3112, viewer: 3113 }, + health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" }, + circuitBreaker: null, + functionMetrics: [], + provider: "llm", + embeddingProvider: "embeddings", + flags: [], + index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 }, + graph: null, + graphExtractionEnabled: false, + ...overrides, + }; +} + +const cleanLeg = { + lastSavedAt: "2026-09-25T11:59:00.000Z", + lastError: null, + lastErrorAt: null, + dirtySince: null, + serializedChars: 1000, +}; + +describe("status reports index persistence", () => { + it("is quiet when both legs saved cleanly", () => { + const report = evaluateStatus( + statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }), + ); + expect(report.problems).toEqual([]); + expect(report.indexPersistence?.vector?.serializedChars).toBe(1000); + }); + + it("reports a failing leg as an error", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" }, + }, + }), + ); + expect(report.status).toBe("error"); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]); + expect(report.problems[0].message).toContain("vector"); + }); + + it("warns when the vector index passes 80% of the Node.js string limit", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) }, + }, + }), + ); + expect(report.status).toBe("warn"); + expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]); + expect(report.problems[0].message).toContain("85%"); + }); + + it("warns when unsaved changes are older than twice the save interval", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" }, + vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" }, + }, + }), + ); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]); + expect(report.problems[0].message).toContain("BM25"); + }); + + it("shows save state and vector size on the status page", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" }, + vector: { ...cleanLeg, serializedChars: 53_687_089 }, + }, + }), + ); + const html = renderStatusHtml(report, "n"); + expect(html).toContain(""); + expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)"); + }); +}); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index ba47c3ef72..802084bd2d 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -670,7 +670,7 @@ describe("IndexPersistence", () => { it("scheduleSave debounces multiple calls", async () => { const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 }); persistence.scheduleSave(); persistence.scheduleSave(); @@ -721,7 +721,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); let unhandled = false; const onUnhandled = () => { @@ -751,7 +751,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); await expect(persistence.save()).resolves.toBeUndefined(); }); From 62b5dc37cb6292857e59fc292856ffcffb2ea882 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Fri, 25 Sep 2026 19:22:27 +0530 Subject: [PATCH 2/7] perf(index): store vectors in fixed buckets and rebuild BM25 from content The vector index was persisted by serializing every vector into one JSON string, cutting that string into shards and writing a new generation of all shards on every save. Node.js cannot hold a string longer than 536,870,888 characters, which a 2,048-dimension index reaches near 49,000 vectors; past it the index can no longer be saved or loaded, silently. Each save also rewrote the whole index. Vectors now live in a fixed set of state scopes (256 by default, AGENTMEMORY_VECTOR_BUCKETS), chosen by a stable hash of the id, with one key per vector holding its id, session and base64 embedding. The index tracks which ids were added or removed since the last save, and a save writes or deletes only those keys, then updates a small metadata record. Loading lists the bucket scopes one at a time. Nothing builds a string of the whole index, and scope names never change between saves. One key per vector, rather than one packed value per bucket: the iii file store rewrites a whole scope when any key in it changes, so both layouts cost the same on disk per flush (one bucket scope, 1/256 of the index). Per-key writes avoid re-encoding a whole bucket in the worker for every changed vector, a single added vector is exactly one state write, and a failed write retries only that vector. BM25 is no longer persisted. It is rebuilt at boot from stored observations and latest memories, the same documents the live paths index. A 3,000-document rebuild takes about 120 ms in the unit test; real boots add the engine round-trips for one list per session, and the boot log now reports the rebuild time. The rebuild also drops entries for content that no longer exists. Documents added after the last vector save that have no vector are embedded in the background; when no vector state was ever saved, every document without a vector is. Existing installs migrate on first boot: the old sharded or monolithic vector snapshot is read once, written into buckets, and only then are the old vector and BM25 shards and manifests removed. If the old snapshot cannot be read it is left untouched. If writing the buckets fails, the old snapshot stays and the next save finishes the job. No vector is lost and nothing is re-embedded. Changing the bucket count moves vectors to their new buckets on the next boot. The dimension-mismatch guard is unchanged; when AGENTMEMORY_DROP_STALE_INDEX discards persisted vectors, the next save now deletes them from storage too. The status page drops the string-size warning, which no longer applies, and shows the bucket count and unsaved vector changes. --- .env.example | 1 + plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/config.ts | 7 + src/functions/search.ts | 87 ++ src/functions/status.ts | 59 +- src/index.ts | 112 +- src/state/index-persistence.ts | 617 +++++----- src/state/vector-index.ts | 55 +- test/index-keyword-rebuild.test.ts | 187 +++ test/index-persistence-throttle.test.ts | 196 ++-- test/index-persistence.test.ts | 1031 +++++------------ 11 files changed, 1033 insertions(+), 1322 deletions(-) create mode 100644 test/index-keyword-rebuild.test.ts diff --git a/.env.example b/.env.example index 4b9deb0b60..f42dd7f272 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,7 @@ # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). # AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. +# AGENTMEMORY_VECTOR_BUCKETS=256 # Number of state scopes the vector index is spread across; a save rewrites only the buckets it touched. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 5fe071567b..abbf39e9f7 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 45 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -46,6 +46,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_TOOLS` - `AGENTMEMORY_URL` - `AGENTMEMORY_USE_DOCKER` +- `AGENTMEMORY_VECTOR_BUCKETS` - `AGENTMEMORY_VERBOSE` - `AGENTMEMORY_VIEWER_HOST` - `AGENTMEMORY_VIEWER_URL` diff --git a/src/config.ts b/src/config.ts index ac82bd445f..ae9fd19f29 100644 --- a/src/config.ts +++ b/src/config.ts @@ -477,6 +477,13 @@ export function getIndexSaveIntervalMs(): number { return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; } +export const VECTOR_BUCKETS_DEFAULT = 256; + +export function getVectorBucketCount(): number { + const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKETS"], VECTOR_BUCKETS_DEFAULT); + return raw > 0 && raw <= 65_536 ? raw : VECTOR_BUCKETS_DEFAULT; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/search.ts b/src/functions/search.ts index 8abce07f45..dc834502c7 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -399,6 +399,93 @@ export async function rebuildIndex(kv: StateKV): Promise { return indexed } +export type VectorBackfillJob = { + id: string + sessionId: string + text: string + context: { kind: "memory" | "observation" | "synthetic"; logId: string } +} + +export async function rebuildKeywordIndex( + kv: StateKV, + vectorBackfillSince?: string | null, +): Promise<{ documents: number; vectorJobs: VectorBackfillJob[] }> { + const idx = getSearchIndex() + idx.clear() + memoryIndexReady = false + const vi = vectorIndex + const backfill = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined + const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN + const vectorJobs: VectorBackfillJob[] = [] + const consider = ( + id: string, + sessionId: string, + text: string, + timestamp: string | undefined, + kind: "memory" | "observation", + ): void => { + if (!backfill || vi?.has(id)) return + if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return + vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } }) + } + + let documents = 0 + let memoriesLoaded = false + try { + const memories = await kv.list(KV.memories) + memoriesLoaded = true + for (const memory of memories) { + if (memory.isLatest === false) continue + if (!memory.title || !memory.content) continue + idx.add(memoryToObservation(memory)) + consider(memory.id, memory.sessionIds?.[0] ?? 'memory', memory.title + ' ' + memory.content, memory.createdAt, "memory") + documents++ + } + } catch (err) { + logger.warn('rebuildKeywordIndex: failed to load memories', { + error: err instanceof Error ? err.message : String(err), + }) + } + + const sessions = await kv.list(KV.sessions) + const failedSessions: string[] = [] + for (let batch = 0; batch < sessions.length; batch += 10) { + const chunk = sessions.slice(batch, batch + 10) + const results = await Promise.all( + chunk.map(async (s) => { + try { + return await kv.list(KV.observations(s.id)) + } catch { + failedSessions.push(s.id) + return [] as CompressedObservation[] + } + }) + ) + for (const obs of results.flat()) { + if (!obs.title || !obs.narrative) continue + idx.add(obs) + consider(obs.id, obs.sessionId, obs.title + ' ' + obs.narrative, obs.timestamp, "observation") + documents++ + } + } + if (failedSessions.length > 0) { + logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions }) + } + if (memoriesLoaded) memoryIndexReady = true + return { documents, vectorJobs } +} + +export async function backfillVectors(jobs: VectorBackfillJob[]): Promise { + const batchSize = getRebuildEmbedBatchSize() + let added = 0 + for (let offset = 0; offset < jobs.length; offset += batchSize) { + const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize)) + added += ok + } + if (added > 0) scheduleIndexSave() + return added +} + export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction( 'mem::search', diff --git a/src/functions/status.ts b/src/functions/status.ts index 0b3ab5310a..db1d49d789 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -1,4 +1,4 @@ -import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js"; +import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js"; export type StatusLevel = "ok" | "info" | "warn" | "error"; @@ -85,7 +85,6 @@ export interface StatusReport { const FAILURE_RATE_THRESHOLD = 0.2; const FAILURE_MIN_CALLS = 5; const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60; -const VECTOR_STRING_WARN_RATIO = 0.8; const LEVEL_RANK: Record = { ok: 0, info: 1, warn: 2, error: 3 }; const FUNCTION_FIXES: Record = { @@ -174,38 +173,23 @@ export function evaluateStatus(input: StatusInputs): StatusReport { } const persistence = input.indexPersistence ?? null; - if (persistence) { - const legs: Array<[string, IndexLegStatus | null]> = [ - ["BM25", persistence.bm25], - ["vector", persistence.vector], - ]; - for (const [label, leg] of legs) { - if (!leg) continue; - if (leg.lastError) { - problems.push({ - level: "error", - code: "index-save-failing", - message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`, - fix: "Check the server log for the failing state write. The next save retries automatically.", - }); - } - const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined); - if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { - problems.push({ - level: "warn", - code: "index-save-stale", - message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, - fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", - }); - } + const vectorLeg = persistence?.vector ?? null; + if (persistence && vectorLeg) { + if (vectorLeg.lastError) { + problems.push({ + level: "error", + code: "index-save-failing", + message: `The vector index could not be saved: ${vectorLeg.lastError}. Search keeps working from memory, but vectors added since the last save are lost on restart.`, + fix: "Check the server log for the failing state write. The next save retries automatically.", + }); } - const vectorChars = persistence.vector?.serializedChars ?? null; - if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) { + const dirtyAge = secondsBetween(input.now, vectorLeg.dirtySince ?? undefined); + if (!vectorLeg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { problems.push({ level: "warn", - code: "vector-index-near-string-limit", - message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`, - fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.", + code: "index-save-stale", + message: `The vector index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, + fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", }); } } @@ -300,16 +284,13 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string { function indexPersistenceRows(report: StatusReport): string { const persistence = report.indexPersistence; if (!persistence) return ""; - let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25))); + let rows = row("BM25 index", "rebuilt from stored content at boot"); if (persistence.vector) { rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector))); - const chars = persistence.vector.serializedChars; - if (chars !== null) { - rows += row( - "Vector index size", - escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`), - ); - } + rows += row( + "Vector storage", + escapeHtml(`${persistence.buckets} buckets, ${persistence.pendingChanges} unsaved changes`), + ); } return rows; } diff --git a/src/index.ts b/src/index.ts index ff465afe46..23c11a2cf8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,6 @@ import { createImageEmbeddingProvider, } from "./providers/index.js"; import { StateKV } from "./state/kv.js"; -import { KV } from "./state/schema.js"; import { VectorIndex } from "./state/vector-index.js"; import { HybridSearch } from "./state/hybrid-search.js"; import { IndexPersistence } from "./state/index-persistence.js"; @@ -35,8 +34,8 @@ import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; import { registerCompressFunction } from "./functions/compress.js"; import { registerSearchFunction, - rebuildIndex, - reconcileIndex, + backfillVectors, + rebuildKeywordIndex, getSearchIndex, setVectorIndex, setEmbeddingProvider, @@ -395,23 +394,13 @@ async function main() { const healthMonitor = registerHealthMonitor(sdk, kv); - const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex); - // Wire the persistence hook so delete paths can flush BM25/vector - // index mutations to disk. Without this, an in-memory remove can be - // lost across a hard process exit and the persisted snapshot - // restores the deleted entry at next boot. + const indexPersistence = new IndexPersistence(kv, vectorIndex); setIndexPersistence(indexPersistence); const loaded = await indexPersistence.load().catch((err) => { - console.warn(`[agentmemory] Failed to load persisted index:`, err); + console.warn(`[agentmemory] Failed to load persisted vector index:`, err); return null; }); - if (loaded?.bm25 && loaded.bm25.size > 0) { - bm25Index.restoreFrom(loaded.bm25); - bootLog( - `Loaded persisted BM25 index (${bm25Index.size} docs)`, - ); - } if (loaded?.vector && vectorIndex && loaded.vector.size > 0) { // Persisted vectors carry whatever dimension the provider had when // they were written. If the active provider declares a different @@ -436,6 +425,7 @@ async function main() { const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); const dropStale = isDropStaleIndexEnabled(); if (dropStale) { + for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId); console.warn( `[agentmemory] Persisted vector index has ${mismatches.length} of ` + `${loaded.vector.size} vectors with the wrong dimension. Active ` + @@ -466,79 +456,29 @@ async function main() { } } - const needsRebuild = bm25Index.size === 0; - - if (needsRebuild) { - // Fire-and-forget. rebuildIndex iterates every observation across - // every session and AWAITS an embedding-provider call per record. - // On a large corpus + rate-limited embedding endpoint that can - // take HOURS; awaiting it here blocks every subsequent boot step - // (including startViewerServer below, leaving the viewer port - // unbound for the duration). The index lazily fills in over time - // and search degrades gracefully — partial coverage > no viewer - // for hours. Errors still surface via the inner .catch. - void rebuildIndex(kv) - .then((indexCount) => { - if (indexCount > 0) { - bootLog(`Search index rebuilt: ${indexCount} entries`); - indexPersistence.scheduleSave(); - } - }) - .catch((err) => { - console.warn(`[agentmemory] Failed to rebuild search index:`, err); - }); - } else { - // Backfill memories into BM25 for users upgrading from <0.9.5: prior - // versions of mem::remember never indexed memories, so the persisted - // BM25 covers observations only and `memory_smart_search` returns - // empty for everything saved via memory_save (#257). Walk KV.memories - // and add the ones missing from the restored index. Idempotent on - // re-runs because SearchIndex.has() short-circuits already-indexed - // ids. - try { - const memories = await kv.list(KV.memories); - let backfilled = 0; - for (const memory of memories) { - if (memory.isLatest === false) continue; - if (!memory.title || !memory.content) continue; - if (bm25Index.has(memory.id)) continue; - bm25Index.add({ - id: memory.id, - sessionId: memory.sessionIds?.[0] ?? "memory", - timestamp: memory.createdAt, - type: "decision", - title: memory.title, - facts: [memory.content], - narrative: memory.content, - concepts: memory.concepts, - files: memory.files, - importance: memory.strength, + const vectorBackfillSince = + !loaded || loaded.state === "unavailable" + ? undefined + : loaded.state === "none" + ? null + : loaded.savedAt; + const keywordStart = Date.now(); + try { + const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince); + bootLog( + `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`, + ); + if (keyword.vectorJobs.length > 0) { + void backfillVectors(keyword.vectorJobs) + .then((count) => { + if (count > 0) bootLog(`Vector index backfilled: ${count} entries`); + }) + .catch((err) => { + console.warn(`[agentmemory] Failed to backfill vectors:`, err); }); - backfilled++; - } - if (backfilled > 0) { - bootLog( - `Backfilled ${backfilled} memories into BM25 (legacy index gap)`, - ); - indexPersistence.scheduleSave(); - } - } catch (err) { - console.warn( - `[agentmemory] Failed to backfill memories into BM25:`, - err, - ); } - void reconcileIndex(kv) - .then((count) => { - if (count > 0) { - bootLog( - `Search index reconciled: ${count} observations missing from the persisted snapshot were re-indexed`, - ); - } - }) - .catch((err) => { - console.warn(`[agentmemory] Failed to reconcile search index:`, err); - }); + } catch (err) { + console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err); } // Ready / Endpoints lines are emitted via `bootLog` so they're diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index a77d66ed30..4658e44371 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -1,30 +1,21 @@ -import { SearchIndex } from "./search-index.js"; -import { VectorIndex } from "./vector-index.js"; +import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js"; import type { StateKV } from "./kv.js"; -import { KV, generateId } from "./schema.js"; +import { KV } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; -import { getIndexSaveIntervalMs } from "../config.js"; +import { getIndexSaveIntervalMs, getVectorBucketCount } from "../config.js"; -export const V8_MAX_STRING_CHARS = 536_870_888; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; -const BM25_KEY = "data"; -const BM25_MANIFEST_KEY = "data:manifest"; -const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`; -const VECTOR_KEY = "vectors"; -const VECTOR_MANIFEST_KEY = "vectors:manifest"; -const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`; -const INDEX_SHARD_KEY = "data"; -const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; - -// mem:audit exists to record structural deletions of user data — that is -// the policy stated at the top of src/functions/audit.ts. Index shard -// writes and manifest publishes remove no user rows, so they fall outside -// it, yet a single save() emits three of them: on a real store they -// reached 59876 of 84028 entries (71%), which is what makes the audit log -// slow to query and bloats startup. Off by default; set -// AGENTMEMORY_AUDIT_INDEX_PERSIST=1 when debugging index persistence. +const LEGACY_BM25_KEY = "data"; +const LEGACY_BM25_MANIFEST_KEY = "data:manifest"; +const LEGACY_VECTOR_KEY = "vectors"; +const LEGACY_VECTOR_MANIFEST_KEY = "vectors:manifest"; +const VECTOR_META_KEY = "vectors:meta"; +const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`; +const WRITE_CONCURRENCY = 32; +const LOAD_CONCURRENCY = 8; + function auditIndexPersistEnabled(): boolean { const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; if (!raw) return false; @@ -39,45 +30,59 @@ type IndexShardManifest = { chars: number; }; +type VectorMeta = { + v: 2; + buckets: number; + savedAt: string; + count: number; +}; + +type PersistedVector = { + id: string; + s: string; + e: string; +}; + type IndexPersistenceOptions = { - shardChars?: number; - createGeneration?: () => string; saveIntervalMs?: number; + buckets?: number; now?: () => number; }; -export type IndexLeg = "bm25" | "vector"; - export interface IndexLegStatus { lastSavedAt: string | null; lastError: string | null; lastErrorAt: string | null; dirtySince: string | null; - serializedChars: number | null; } export interface IndexPersistenceStatus { saveIntervalMs: number; saving: boolean; - bm25: IndexLegStatus; + buckets: number; + pendingChanges: number; vector: IndexLegStatus | null; } -function emptyLegStatus(): IndexLegStatus { - return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null }; +export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable"; + +export interface VectorLoadResult { + vector: VectorIndex | null; + state: VectorLoadState; + savedAt: string | null; } -function shardChars(options: IndexPersistenceOptions): number { - const configured = options.shardChars; - if (typeof configured !== "number" || !Number.isFinite(configured)) { - return DEFAULT_INDEX_SHARD_CHARS; +export function vectorBucketOf(id: string, buckets: number): number { + let hash = 0x811c9dc5; + for (let i = 0; i < id.length; i++) { + hash ^= id.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; } - const wholeChars = Math.floor(configured); - return wholeChars >= 1 ? wholeChars : DEFAULT_INDEX_SHARD_CHARS; + return hash % buckets; } -function createIndexGeneration(): string { - return generateId("idx"); +export function vectorBucketScope(bucket: number): string { + return `${VECTOR_BUCKET_SCOPE_PREFIX}${String(bucket).padStart(4, "0")}`; } function statePath(scope: string, key: string): string { @@ -88,6 +93,13 @@ function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } +function generationTime(generation: string | undefined): string | null { + const stamp = generation?.split("_")[1]; + if (!stamp) return null; + const ms = parseInt(stamp, 36); + return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : null; +} + function isValidShardDescriptor( shard: unknown, ): shard is IndexShardManifest["shards"][number] { @@ -99,31 +111,50 @@ function isValidShardDescriptor( typeof candidate.key === "string" && candidate.key.length > 0 && Number.isInteger(candidate.chars) && - candidate.chars >= 0 + (candidate.chars as number) >= 0 ); } +function isPersistedVector(row: unknown): row is PersistedVector { + if (!row || typeof row !== "object") return false; + const candidate = row as Partial; + return typeof candidate.id === "string" && typeof candidate.s === "string" && typeof candidate.e === "string"; +} + +async function inBatches(items: T[], size: number, run: (item: T) => Promise): Promise { + const failures: unknown[] = []; + for (let offset = 0; offset < items.length; offset += size) { + const results = await Promise.allSettled(items.slice(offset, offset + size).map(run)); + for (const result of results) { + if (result.status === "rejected") failures.push(result.reason); + } + } + return failures; +} + +function emptyLegStatus(): IndexLegStatus { + return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null }; +} + export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = new Map(); + private lastFailureLogAt = 0; private running: Promise | null = null; private queued: Promise | null = null; private stopped = false; private lastSaveAt: number; private dirtyEpoch = 0; private markedDuringRunAt: number | null = null; - private legs: Record = { - bm25: emptyLegStatus(), - vector: emptyLegStatus(), - }; + private metaWritten = false; + private leg: IndexLegStatus = emptyLegStatus(); private readonly saveIntervalMs: number; + private buckets: number; private readonly now: () => number; constructor( private kv: StateKV, - private bm25: SearchIndex, private vector: VectorIndex | null, - private options: IndexPersistenceOptions = {}, + options: IndexPersistenceOptions = {}, ) { this.now = options.now ?? Date.now; const interval = options.saveIntervalMs; @@ -131,22 +162,25 @@ export class IndexPersistence { typeof interval === "number" && Number.isFinite(interval) && interval > 0 ? interval : getIndexSaveIntervalMs(); + const buckets = options.buckets; + this.buckets = + typeof buckets === "number" && Number.isInteger(buckets) && buckets > 0 + ? buckets + : getVectorBucketCount(); this.lastSaveAt = this.now(); } scheduleSave(): void { - if (this.stopped) return; + if (this.stopped || !this.vector) return; const now = this.now(); this.dirtyEpoch++; if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now; - for (const leg of this.activeLegs()) { - if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString(); - } + if (this.leg.dirtySince === null) this.leg.dirtySince = new Date(now).toISOString(); if (this.timer) return; const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now); this.timer = setTimeout(() => { this.timer = null; - this.save().catch((err) => this.logFailure("bm25", err)); + this.save().catch((err) => this.logFailure(err)); }, delay); } @@ -168,34 +202,42 @@ export class IndexPersistence { return { saveIntervalMs: this.saveIntervalMs, saving: this.running !== null, - bm25: { ...this.legs.bm25 }, - vector: this.vector ? { ...this.legs.vector } : null, + buckets: this.buckets, + pendingChanges: this.vector?.pendingChanges ?? 0, + vector: this.vector ? { ...this.leg } : null, }; } - async load(): Promise<{ - bm25: SearchIndex | null; - vector: VectorIndex | null; - }> { - let bm25: SearchIndex | null = null; - let vector: VectorIndex | null = null; + stop(): void { + this.stopped = true; + this.clearTimer(); + } - const bm25Data = await this.loadBm25Data(); - if (bm25Data && typeof bm25Data === "string") { - bm25 = SearchIndex.deserialize(bm25Data); + async load(): Promise { + if (!this.vector) { + await this.removeLegacyBm25Snapshot(); + return { vector: null, state: "none", savedAt: null }; } - const vecData = await this.loadVectorData(); - if (vecData && typeof vecData === "string") { - vector = VectorIndex.deserialize(vecData); + let meta: VectorMeta | null; + try { + meta = await this.kv.get(KV.bm25Index, VECTOR_META_KEY); + } catch (err) { + logger.warn("index persistence: vector metadata read failed", { message: errorMessage(err) }); + return { vector: null, state: "unavailable", savedAt: null }; } - return { bm25, vector }; - } + if (meta && meta.v === 2 && Number.isInteger(meta.buckets) && meta.buckets > 0) { + const loaded = await this.loadBuckets(meta.buckets); + if (!loaded) return { vector: null, state: "unavailable", savedAt: null }; + this.metaWritten = true; + if (meta.buckets !== this.buckets) await this.moveBuckets(loaded, meta.buckets); + await this.removeLegacyVectorSnapshotIfPresent(); + await this.removeLegacyBm25Snapshot(); + return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null }; + } - stop(): void { - this.stopped = true; - this.clearTimer(); + return this.migrateLegacy(); } private clearTimer(): void { @@ -205,10 +247,6 @@ export class IndexPersistence { } } - private activeLegs(): IndexLeg[] { - return this.vector ? ["bm25", "vector"] : ["bm25"]; - } - private startRun(): Promise { const run = this.runSave().finally(() => { if (this.running === run) this.running = null; @@ -218,176 +256,215 @@ export class IndexPersistence { } private async runSave(): Promise { + const vector = this.vector; + if (!vector) return; const epoch = this.dirtyEpoch; this.markedDuringRunAt = null; this.lastSaveAt = this.now(); - await this.saveLeg("bm25", epoch, async () => { - const serialized = this.bm25.serialize(); - this.legs.bm25.serializedChars = serialized.length; - await this.saveBm25Index(serialized); - }); - const vector = this.vector; - if (vector) { - await this.saveLeg("vector", epoch, async () => { - const serialized = vector.serialize(); - this.legs.vector.serializedChars = serialized.length; - await this.saveVectorIndex(serialized); - }); - } - } - - private async saveLeg(leg: IndexLeg, epoch: number, run: () => Promise): Promise { - const status = this.legs[leg]; try { - await run(); - status.lastSavedAt = new Date(this.now()).toISOString(); - status.lastError = null; - status.lastErrorAt = null; - if (this.dirtyEpoch === epoch) { - status.dirtySince = null; + await this.writeChanges(vector); + this.leg.lastSavedAt = new Date(this.now()).toISOString(); + this.leg.lastError = null; + this.leg.lastErrorAt = null; + if (this.dirtyEpoch === epoch || vector.pendingChanges === 0) { + this.leg.dirtySince = null; } else if (this.markedDuringRunAt !== null) { - status.dirtySince = new Date(this.markedDuringRunAt).toISOString(); + this.leg.dirtySince = new Date(this.markedDuringRunAt).toISOString(); } } catch (err) { - status.lastError = errorMessage(err); - status.lastErrorAt = new Date(this.now()).toISOString(); - if (status.dirtySince === null) status.dirtySince = status.lastErrorAt; - this.logFailure(leg, err); + this.leg.lastError = errorMessage(err); + this.leg.lastErrorAt = new Date(this.now()).toISOString(); + if (this.leg.dirtySince === null) this.leg.dirtySince = this.leg.lastErrorAt; + this.logFailure(err); } } - private logFailure(leg: IndexLeg, err: unknown): void { - const now = this.now(); - if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt.set(leg, now); - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, { - code, - message, - hint: - code === "TIMEOUT" - ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save" - : undefined, + private async writeChanges(vector: VectorIndex): Promise { + const changes = vector.takeChanges(); + if (changes.size === 0 && this.metaWritten) return; + const failed = new Map(); + const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => { + const scope = vectorBucketScope(vectorBucketOf(id, this.buckets)); + const entry = present ? vector.get(id) : undefined; + try { + if (entry) { + await this.kv.set(scope, id, { + id, + s: entry.sessionId, + e: float32ToBase64(entry.embedding), + }); + } else { + await this.kv.delete(scope, id); + } + } catch (err) { + failed.set(id, present); + throw err; + } }); + if (failures.length > 0) { + vector.returnChanges(failed); + throw new Error( + `${failures.length} of ${changes.size} vector writes failed: ${errorMessage(failures[0])}`, + ); + } + await this.kv.set(KV.bm25Index, VECTOR_META_KEY, { + v: 2, + buckets: this.buckets, + savedAt: new Date(this.now()).toISOString(), + count: vector.size, + }); + this.metaWritten = true; } - private async saveBm25Index(serialized: string): Promise { - await this.saveShardedIndex( - serialized, - BM25_MANIFEST_KEY, - BM25_KEY, - BM25_SHARD_SCOPE_PREFIX, - ); + private async loadBuckets(buckets: number): Promise { + const loaded = new VectorIndex(); + const bucketIds = Array.from({ length: buckets }, (_, bucket) => bucket); + const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => { + const rows = await this.kv.list(vectorBucketScope(bucket)); + for (const row of rows) { + if (!isPersistedVector(row)) continue; + try { + loaded.loadPersisted(row.id, row.s, base64ToFloat32(row.e)); + } catch { + continue; + } + } + }); + if (failures.length > 0) { + logger.warn("index persistence: vector bucket read failed", { + failed: failures.length, + message: errorMessage(failures[0]), + }); + return null; + } + return loaded; } - private async saveVectorIndex(serialized: string): Promise { - await this.saveShardedIndex( - serialized, - VECTOR_MANIFEST_KEY, - VECTOR_KEY, - VECTOR_SHARD_SCOPE_PREFIX, + private async moveBuckets(vector: VectorIndex, fromBuckets: number): Promise { + const moves = [...vector.entries()].filter( + ([id]) => vectorBucketOf(id, fromBuckets) !== vectorBucketOf(id, this.buckets), ); + const failures = await inBatches(moves, WRITE_CONCURRENCY, async ([id, entry]) => { + await this.kv.set(vectorBucketScope(vectorBucketOf(id, this.buckets)), id, { + id, + s: entry.sessionId, + e: float32ToBase64(entry.embedding), + }); + await this.kv.delete(vectorBucketScope(vectorBucketOf(id, fromBuckets)), id); + }); + if (failures.length > 0) { + logger.warn("index persistence: moving vectors to the new bucket count failed; keeping the old count", { + failed: failures.length, + message: errorMessage(failures[0]), + }); + this.buckets = fromBuckets; + return; + } + await this.kv.set(KV.bm25Index, VECTOR_META_KEY, { + v: 2, + buckets: this.buckets, + savedAt: new Date(this.now()).toISOString(), + count: vector.size, + }); } - private async saveShardedIndex( - serialized: string, - manifestKey: string, - legacyKey: string, - scopePrefix: string, - ): Promise { - const previous = await this.kv - .get(KV.bm25Index, manifestKey) - .catch(() => null); - const generation = - this.options.createGeneration?.() ?? createIndexGeneration(); - const chunkChars = shardChars(this.options); - const shards: IndexShardManifest["shards"] = []; - const chunks: string[] = []; - - for (let offset = 0; offset < serialized.length; offset += chunkChars) { - const shardIndex = shards.length; - const scope = `${scopePrefix}${generation}:${String(shardIndex).padStart( - 5, - "0", - )}`; - const chunk = serialized.slice(offset, offset + chunkChars); - shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length }); - chunks.push(chunk); + private async migrateLegacy(): Promise { + const manifest = await this.readIndexValue( + KV.bm25Index, + LEGACY_VECTOR_MANIFEST_KEY, + "vector", + "manifest", + ); + const legacyKey = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy"); + if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null }; + const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0); + if (!legacyPresent) { + await this.removeLegacyBm25Snapshot(); + return { vector: null, state: "none", savedAt: null }; } - const writeResults = await Promise.allSettled( - shards.map(async (shard, index) => { - const chunk = chunks[index] ?? ""; - await this.kv.set(shard.scope, shard.key, chunk); - await this.auditIndexPersistence("shard_write", [ - statePath(shard.scope, shard.key), - ], { - scope: shard.scope, - key: shard.key, - manifestKey, - generation, - chars: chunk.length, - }); - }), - ); - const failedWrite = writeResults.find( - (result): result is PromiseRejectedResult => result.status === "rejected", - ); - if (failedWrite) { - await this.deleteShards(shards, "shard_write_rollback"); - throw failedWrite.reason; + const data = await this.loadShardedData(LEGACY_VECTOR_KEY, LEGACY_VECTOR_MANIFEST_KEY, "vector"); + if (typeof data !== "string") { + logger.warn("index persistence: legacy vector snapshot is unreadable; keeping it and skipping migration"); + return { vector: null, state: "unavailable", savedAt: null }; } + const legacy = VectorIndex.deserialize(data); + const savedAt = + (manifest.value && typeof manifest.value === "object" ? generationTime(manifest.value.generation) : null) ?? + new Date(this.now()).toISOString(); - const nextManifest: IndexShardManifest = { - v: 1, - generation, - shards, - chars: serialized.length, - }; + const staging = new VectorIndex(); + for (const [id, entry] of legacy.entries()) staging.add(id, entry.sessionId, entry.embedding); try { - await this.kv.set( - KV.bm25Index, - manifestKey, - nextManifest, - ); - await this.auditIndexPersistence("manifest_publish", [ - statePath(KV.bm25Index, manifestKey), - ], { - manifestKey, - generation, - chars: serialized.length, - shards: shards.length, - result: "committed", - }); + await this.writeChanges(staging); } catch (err) { - if (await this.isManifestPublished(manifestKey, nextManifest)) { - await this.auditIndexPersistence("manifest_publish", [ - statePath(KV.bm25Index, manifestKey), - ], { - manifestKey, - generation, - chars: serialized.length, - shards: shards.length, - result: "committed_after_error", - error: errorMessage(err), - }); - } else { - await this.deleteShards(shards, "manifest_publish_rollback"); - } - throw err; + logger.warn("index persistence: migrating the legacy vector snapshot failed; it stays in place and the next save retries", { + message: errorMessage(err), + }); + legacy.markAllChanged(); + return { vector: legacy, state: "unavailable", savedAt }; } - await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup"); - if (previous?.v === 1 && Array.isArray(previous.shards)) { - const currentShardIds = new Set( - shards.map((shard) => `${shard.scope}\0${shard.key}`), - ); - for (const shard of previous.shards) { - if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue; - await this.deleteShards([shard], "previous_generation_cleanup"); - } + await this.removeLegacyVectorSnapshot(manifest.value); + await this.removeLegacyBm25Snapshot(); + await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], { + vectors: legacy.size, + buckets: this.buckets, + }); + logger.info("index persistence: migrated the vector index to bucketed storage", { + vectors: legacy.size, + buckets: this.buckets, + }); + const migrated = new VectorIndex(); + for (const [id, entry] of legacy.entries()) migrated.loadPersisted(id, entry.sessionId, entry.embedding); + return { vector: migrated, state: "migrated", savedAt }; + } + + private async removeLegacyVectorSnapshot(manifest: IndexShardManifest | null): Promise { + if (manifest && Array.isArray(manifest.shards)) { + await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_vector_cleanup"); } + await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "legacy_vector_cleanup"); + await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_KEY, "legacy_vector_cleanup"); + } + + private async removeLegacyVectorSnapshotIfPresent(): Promise { + const manifest = await this.kv + .get(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY) + .catch(() => undefined); + const legacy = await this.kv.get(KV.bm25Index, LEGACY_VECTOR_KEY).catch(() => undefined); + if (manifest === undefined || legacy === undefined) return; + if (manifest == null && legacy == null) return; + await this.removeLegacyVectorSnapshot(manifest); + } + + private async removeLegacyBm25Snapshot(): Promise { + const manifest = await this.kv + .get(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY) + .catch(() => undefined); + const legacy = await this.kv.get(KV.bm25Index, LEGACY_BM25_KEY).catch(() => undefined); + if (manifest === undefined || legacy === undefined) return; + if (manifest == null && legacy == null) return; + if (manifest && Array.isArray(manifest.shards)) { + await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_bm25_cleanup"); + } + if (manifest != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "legacy_bm25_cleanup"); + if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup"); + } + + private logFailure(err: unknown): void { + const now = this.now(); + if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt = now; + const code = (err as { code?: string })?.code; + logger.warn("index persistence: failed to save the vector index", { + code, + message: errorMessage(err), + hint: + code === "TIMEOUT" + ? "iii-engine state::set timed out; unsaved vectors stay in memory and retry on the next save" + : undefined, + }); } private async auditIndexPersistence( @@ -396,20 +473,13 @@ export class IndexPersistence { details: Record, ): Promise { if (!auditIndexPersistEnabled()) return; - await safeAudit( - this.kv, - "index_persist", - INDEX_PERSISTENCE_FUNCTION_ID, - targetIds, - { action, ...details }, - ); + await safeAudit(this.kv, "index_persist", INDEX_PERSISTENCE_FUNCTION_ID, targetIds, { + action, + ...details, + }); } - private async deleteKey( - scope: string, - key: string, - reason: string, - ): Promise { + private async deleteKey(scope: string, key: string, reason: string): Promise { let result = "deleted"; let error: string | undefined; try { @@ -427,81 +497,17 @@ export class IndexPersistence { }); } - private async deleteShards( - shards: IndexShardManifest["shards"], - reason: string, - ): Promise { - for (const shard of shards) { - await this.deleteKey(shard.scope, shard.key, reason); - } + private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise { + await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason)); } - private async isManifestPublished( - manifestKey: string, - expected: IndexShardManifest, - ): Promise { - const published = await this.kv - .get(KV.bm25Index, manifestKey) - .catch(() => null); - if ( - published?.v !== 1 || - published.generation !== expected.generation || - published.chars !== expected.chars || - !Array.isArray(published.shards) || - published.shards.length !== expected.shards.length - ) { - return false; - } - return published.shards.every((shard, index) => { - const expectedShard = expected.shards[index]; - if (!expectedShard) return false; - return ( - shard.scope === expectedShard.scope && - shard.key === expectedShard.key && - shard.chars === expectedShard.chars - ); - }); - } - - private async loadBm25Data(): Promise { - return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25"); - } - - private async loadVectorData(): Promise { - return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector"); - } - - private async loadShardedData( - legacyKey: string, - manifestKey: string, - label: string, - ): Promise { - const manifest = await this.readIndexValue( - KV.bm25Index, - manifestKey, - label, - "manifest", - ); + private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise { + const manifest = await this.readIndexValue(KV.bm25Index, manifestKey, label, "manifest"); if (!manifest.ok) return null; - // #797: some iii-state adapters return `undefined` (not `null`) for - // a missing key. The previous `value !== null` check passed - // undefined through to loadManifestData, which then crashed on - // `manifest.v` with TypeError. Treat both null and undefined as - // "no manifest" and fall through to the legacy path. The shape - // check stays so a malformed-but-present row still fails closed. - if ( - manifest.value != null && - typeof manifest.value === "object" - ) { + if (manifest.value != null && typeof manifest.value === "object") { return this.loadManifestData(manifest.value, label); } - - const legacy = await this.readIndexValue( - KV.bm25Index, - legacyKey, - label, - "legacy", - ); + const legacy = await this.readIndexValue(KV.bm25Index, legacyKey, label, "legacy"); if (!legacy.ok) return null; if (legacy.value && typeof legacy.value === "string") return legacy.value; return null; @@ -525,10 +531,7 @@ export class IndexPersistence { } } - private async loadManifestData( - manifest: IndexShardManifest, - label: string, - ): Promise { + private async loadManifestData(manifest: IndexShardManifest, label: string): Promise { if ( manifest.v !== 1 || !Array.isArray(manifest.shards) || @@ -555,10 +558,7 @@ export class IndexPersistence { let chars = 0; for (const { shard, chunk } of loadedShards) { if (typeof chunk !== "string") { - logger.warn(`index persistence: ${label} shard missing`, { - scope: shard.scope, - key: shard.key, - }); + logger.warn(`index persistence: ${label} shard missing`, { scope: shard.scope, key: shard.key }); return null; } if (chunk.length !== shard.chars) { @@ -574,10 +574,7 @@ export class IndexPersistence { chars += chunk.length; } if (chars !== manifest.chars) { - logger.warn(`index persistence: ${label} total length mismatch`, { - expected: manifest.chars, - actual: chars, - }); + logger.warn(`index persistence: ${label} total length mismatch`, { expected: manifest.chars, actual: chars }); return null; } return chunks.join(""); diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index d4b8bda760..2425bef8e1 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -5,13 +5,13 @@ // pool. Same risk on the encode side if the input Float32Array is itself // a sliced view. Reported as a phantom "2048 dimensions on disk" crash // in #455 / #469 / #584 / #587. -function float32ToBase64(arr: Float32Array): string { +export function float32ToBase64(arr: Float32Array): string { return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString( "base64", ); } -function base64ToFloat32(b64: string): Float32Array { +export function base64ToFloat32(b64: string): Float32Array { const buf = Buffer.from(b64, "base64"); return new Float32Array( buf.buffer, @@ -34,16 +34,59 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number { return denom === 0 ? 0 : dot / denom; } +export type VectorEntry = { embedding: Float32Array; sessionId: string }; + export class VectorIndex { - private vectors: Map = - new Map(); + private vectors: Map = new Map(); + private changes: Map = new Map(); add(obsId: string, sessionId: string, embedding: Float32Array): void { this.vectors.set(obsId, { embedding, sessionId }); + this.changes.set(obsId, true); } remove(obsId: string): void { - this.vectors.delete(obsId); + if (this.vectors.delete(obsId)) this.changes.set(obsId, false); + } + + has(obsId: string): boolean { + return this.vectors.has(obsId); + } + + get(obsId: string): VectorEntry | undefined { + return this.vectors.get(obsId); + } + + entries(): IterableIterator<[string, VectorEntry]> { + return this.vectors.entries(); + } + + loadPersisted(obsId: string, sessionId: string, embedding: Float32Array): void { + this.vectors.set(obsId, { embedding, sessionId }); + } + + get pendingChanges(): number { + return this.changes.size; + } + + takeChanges(): Map { + const taken = this.changes; + this.changes = new Map(); + return taken; + } + + returnChanges(changes: Map): void { + for (const [obsId, present] of changes) { + if (!this.changes.has(obsId)) this.changes.set(obsId, present); + } + } + + markRemoved(obsId: string): void { + if (!this.vectors.has(obsId)) this.changes.set(obsId, false); + } + + markAllChanged(): void { + for (const obsId of this.vectors.keys()) this.changes.set(obsId, true); } search( @@ -104,6 +147,7 @@ export class VectorIndex { } clear(): void { + for (const obsId of this.vectors.keys()) this.changes.set(obsId, false); this.vectors.clear(); } @@ -119,6 +163,7 @@ export class VectorIndex { sessionId: entry.sessionId, }); } + this.changes = new Map(other.changes); } serialize(): string { diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.ts new file mode 100644 index 0000000000..4778106bec --- /dev/null +++ b/test/index-keyword-rebuild.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { + backfillVectors, + getSearchIndex, + isMemoryIndexReady, + rebuildKeywordIndex, + setEmbeddingProvider, + setIndexPersistence, + setVectorIndex, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import type { CompressedObservation, EmbeddingProvider, Memory, Session } from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + store, + list: async (scope: string): Promise => Array.from(store.get(scope)?.values() ?? []) as T[], + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + }; +} + +const WORDS = ["auth", "cache", "index", "vector", "session", "memory", "graph", "retry", "queue", "schema", "deploy", "worker"]; + +function makeObs(id: string, sessionId: string, i: number, timestamp: string): CompressedObservation { + const a = WORDS[i % WORDS.length]; + const b = WORDS[(i * 7) % WORDS.length]; + return { + id, + sessionId, + timestamp, + type: "file_edit", + title: `${a} change ${i}`, + facts: [`touched ${b} path`], + narrative: `Updated the ${a} module so ${b} handling works for case ${i}`, + concepts: [a, b], + files: [`src/${a}/${b}.ts`], + importance: 5, + }; +} + +function seed(kv: ReturnType, sessions: number, perSession: number, timestamp: (i: number) => string) { + let n = 0; + for (let s = 0; s < sessions; s++) { + const sessionId = `ses_${s}`; + kv.store.set("mem:sessions", kv.store.get("mem:sessions") ?? new Map()); + kv.store.get("mem:sessions")!.set(sessionId, { + id: sessionId, + project: "p", + cwd: "/tmp", + startedAt: timestamp(0), + status: "completed", + observationCount: perSession, + } satisfies Session); + const scope = `mem:obs:${sessionId}`; + kv.store.set(scope, new Map()); + for (let o = 0; o < perSession; o++) { + const id = `obs_${n}`; + kv.store.get(scope)!.set(id, makeObs(id, sessionId, n, timestamp(n))); + n++; + } + } + const memories = new Map(); + memories.set("mem_1", { + id: "mem_1", + createdAt: timestamp(0), + updatedAt: timestamp(0), + type: "fact", + title: "Deploy uses blue green", + content: "Deploys switch traffic after health checks pass", + concepts: ["deploy"], + files: [], + sessionIds: ["ses_0"], + strength: 7, + version: 1, + isLatest: true, + } as Memory); + memories.set("mem_old", { + id: "mem_old", + createdAt: timestamp(0), + updatedAt: timestamp(0), + type: "fact", + title: "Superseded", + content: "Old version", + concepts: [], + files: [], + sessionIds: [], + strength: 1, + version: 1, + isLatest: false, + } as Memory); + kv.store.set("mem:memories", memories); + kv.store.get("mem:obs:ses_0")!.set("raw_1", { id: "raw_1", sessionId: "ses_0", timestamp: timestamp(0) }); + return n; +} + +function stubProvider(): EmbeddingProvider { + return { + name: "stub", + dimensions: 3, + embed: async () => new Float32Array([0.1, 0.2, 0.3]), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array([0.1, 0.2, 0.3])), + } as EmbeddingProvider; +} + +describe("rebuildKeywordIndex", () => { + afterEach(() => { + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + setIndexPersistence(null); + }); + + it("rebuilds BM25 from stored observations and latest memories, quickly", async () => { + const kv = mockKV(); + const total = seed(kv, 100, 30, () => "2026-09-01T00:00:00.000Z"); + getSearchIndex().add(makeObs("ghost", "ses_gone", 0, "2026-09-01T00:00:00.000Z")); + + const started = performance.now(); + const result = await rebuildKeywordIndex(kv as never); + const elapsed = performance.now() - started; + + expect(total).toBe(3000); + expect(result.documents).toBe(3001); + expect(result.vectorJobs).toEqual([]); + const idx = getSearchIndex(); + expect(idx.size).toBe(3001); + expect(idx.has("ghost")).toBe(false); + expect(idx.has("mem_1")).toBe(true); + expect(idx.has("mem_old")).toBe(false); + expect(idx.has("raw_1")).toBe(false); + expect(idx.search("blue green").map((r) => r.obsId)).toContain("mem_1"); + expect(isMemoryIndexReady()).toBe(true); + expect(elapsed).toBeLessThan(10_000); + }); + + it("queues embeddings only for documents newer than the last vector save", async () => { + const kv = mockKV(); + seed(kv, 2, 5, (i) => (i < 5 ? "2026-09-01T00:00:00.000Z" : "2026-09-20T00:00:00.000Z")); + const vector = new VectorIndex(); + vector.add("obs_6", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + + const result = await rebuildKeywordIndex(kv as never, "2026-09-10T00:00:00.000Z"); + + expect(result.vectorJobs.map((j) => j.id).sort()).toEqual(["obs_5", "obs_7", "obs_8", "obs_9"]); + }); + + it("queues every missing document when no vector state was persisted, and none when storage was unavailable", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + setVectorIndex(new VectorIndex()); + setEmbeddingProvider(stubProvider()); + + const fresh = await rebuildKeywordIndex(kv as never, null); + expect(fresh.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]); + + const unavailable = await rebuildKeywordIndex(kv as never, undefined); + expect(unavailable.vectorJobs).toEqual([]); + }); + + it("backfills queued vectors in batches and schedules a save", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + const vector = new VectorIndex(); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + let scheduled = 0; + setIndexPersistence({ scheduleSave: () => void scheduled++, save: async () => {} }); + + const { vectorJobs } = await rebuildKeywordIndex(kv as never, null); + const added = await backfillVectors(vectorJobs); + + expect(added).toBe(4); + expect(vector.size).toBe(4); + expect(scheduled).toBe(1); + }); +}); diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts index 8a013fa662..09e0572981 100644 --- a/test/index-persistence-throttle.test.ts +++ b/test/index-persistence-throttle.test.ts @@ -1,18 +1,15 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js"; -import { SearchIndex } from "../src/state/search-index.js"; +import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js"; -import type { CompressedObservation } from "../src/types.js"; -const BM25_SCOPE = "mem:index:bm25"; +const META_KEY = "vectors:meta"; function mockKV() { const store = new Map>(); const sets: Array<{ scope: string; key: string }> = []; return { sets, - store, get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, set: async (scope: string, key: string, data: T): Promise => { sets.push({ scope, key }); @@ -27,23 +24,12 @@ function mockKV() { }; } -function obs(id: string, title: string): CompressedObservation { - return { - id, - sessionId: "ses_1", - timestamp: new Date().toISOString(), - type: "file_edit", - title, - facts: [], - narrative: `${title} narrative`, - concepts: [], - files: [], - importance: 5, - }; +function metaSaves(kv: ReturnType): number { + return kv.sets.filter((s) => s.key === META_KEY).length; } -function manifestSaves(kv: ReturnType, key: string): number { - return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length; +function touch(vector: VectorIndex, id: string): void { + vector.add(id, "ses_1", new Float32Array([Math.random(), 0.2, 0.3])); } describe("IndexPersistence save throttling", () => { @@ -59,35 +45,40 @@ describe("IndexPersistence save throttling", () => { }); it("saves at most once per interval however often changes are scheduled", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); - for (let i = 0; i < 50; i++) persistence.scheduleSave(); + for (let i = 0; i < 50; i++) { + touch(vector, `obs_${i}`); + persistence.scheduleSave(); + } await vi.advanceTimersByTimeAsync(59_000); - expect(manifestSaves(kv, "data:manifest")).toBe(0); + expect(metaSaves(kv)).toBe(0); await vi.advanceTimersByTimeAsync(1_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); - for (let i = 0; i < 50; i++) persistence.scheduleSave(); + for (let i = 0; i < 50; i++) { + touch(vector, `obs_more_${i}`); + persistence.scheduleSave(); + } await vi.advanceTimersByTimeAsync(30_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); await vi.advanceTimersByTimeAsync(30_000); - expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(metaSaves(kv)).toBe(2); }); it("an explicit save runs immediately and cancels the pending timer", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); + touch(vector, "obs_1"); persistence.scheduleSave(); await persistence.save(); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); await vi.advanceTimersByTimeAsync(120_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); }); it("never runs two saves at once and coalesces requests made during a save into one", async () => { @@ -108,11 +99,12 @@ describe("IndexPersistence save throttling", () => { return result; }, }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); const first = persistence.save(); + touch(vector, "obs_2"); const second = persistence.save(); const third = persistence.save(); expect(second).toBe(third); @@ -122,42 +114,12 @@ describe("IndexPersistence save throttling", () => { release(); await Promise.all([first, second, third]); - expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(metaSaves(kv)).toBe(2); expect(maxInFlight).toBe(1); expect(persistence.status().saving).toBe(false); }); - it("a failing BM25 leg does not stop the vector leg from saving", async () => { - const failingKv = { - ...kv, - set: async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed"); - return kv.set(scope, key, data); - }, - }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const vector = new VectorIndex(); - vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); - const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 }); - - persistence.scheduleSave(); - await persistence.save(); - - const status = persistence.status(); - expect(status.bm25.lastError).toBe("bm25 manifest write failed"); - expect(status.bm25.dirtySince).not.toBeNull(); - expect(status.vector?.lastError).toBeNull(); - expect(status.vector?.lastSavedAt).not.toBeNull(); - expect(status.vector?.dirtySince).toBeNull(); - expect(status.vector?.serializedChars).toBeGreaterThan(0); - - const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector?.size).toBe(1); - }); - - it("keeps a leg dirty when a change arrives while it is being saved", async () => { + it("keeps the index dirty when a change arrives while it is being saved", async () => { let release: () => void = () => undefined; const gate = new Promise((resolve) => { release = resolve; @@ -173,30 +135,31 @@ describe("IndexPersistence save throttling", () => { return kv.set(scope, key, data); }, }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); const saving = persistence.save(); + touch(vector, "obs_2"); persistence.scheduleSave(); release(); await saving; - expect(persistence.status().bm25.lastSavedAt).not.toBeNull(); - expect(persistence.status().bm25.dirtySince).not.toBeNull(); + expect(persistence.status().vector?.lastSavedAt).not.toBeNull(); + expect(persistence.status().vector?.dirtySince).not.toBeNull(); await persistence.save(); - expect(persistence.status().bm25.dirtySince).toBeNull(); + expect(persistence.status().vector?.dirtySince).toBeNull(); }); it("stop prevents later scheduled saves", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, buckets: 16 }); persistence.stop(); persistence.scheduleSave(); await vi.advanceTimersByTimeAsync(10_000); - expect(manifestSaves(kv, "data:manifest")).toBe(0); + expect(metaSaves(kv)).toBe(0); }); }); @@ -225,78 +188,53 @@ const cleanLeg = { lastError: null, lastErrorAt: null, dirtySince: null, - serializedChars: 1000, }; +function persistenceStatus(vector: typeof cleanLeg | null, pendingChanges = 0) { + return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector }; +} + describe("status reports index persistence", () => { - it("is quiet when both legs saved cleanly", () => { - const report = evaluateStatus( - statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }), - ); + it("is quiet when the vector index saved cleanly", () => { + const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(cleanLeg) })); expect(report.problems).toEqual([]); - expect(report.indexPersistence?.vector?.serializedChars).toBe(1000); + expect(report.indexPersistence?.buckets).toBe(256); }); - it("reports a failing leg as an error", () => { + it("reports a failing vector save as an error", () => { const report = evaluateStatus( statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: cleanLeg, - vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" }, - }, + indexPersistence: persistenceStatus({ + ...cleanLeg, + lastError: "3 of 10 vector writes failed: timed out", + lastErrorAt: "2026-09-25T11:59:30.000Z", + } as never), }), ); expect(report.status).toBe("error"); expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]); - expect(report.problems[0].message).toContain("vector"); - }); - - it("warns when the vector index passes 80% of the Node.js string limit", () => { - const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: cleanLeg, - vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) }, - }, - }), - ); - expect(report.status).toBe("warn"); - expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]); - expect(report.problems[0].message).toContain("85%"); + expect(report.problems[0].message).toContain("3 of 10 vector writes failed"); }); it("warns when unsaved changes are older than twice the save interval", () => { const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" }, - vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" }, - }, - }), + statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" } as never, 4) }), ); expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]); - expect(report.problems[0].message).toContain("BM25"); }); - it("shows save state and vector size on the status page", () => { + it("does not report vector persistence when vector search is off", () => { + const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(null) })); + expect(report.problems).toEqual([]); + }); + + it("shows save state and bucket storage on the status page", () => { const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" }, - vector: { ...cleanLeg, serializedChars: 53_687_089 }, - }, - }), + statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" } as never, 3) }), ); const html = renderStatusHtml(report, "n"); - expect(html).toContain(""); - expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)"); + expect(html).toContain(""); + expect(html).toContain(""); + expect(html).toContain(""); }); }); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 802084bd2d..c469be8134 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1,740 +1,398 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { IndexPersistence } from "../src/state/index-persistence.js"; +import { IndexPersistence, vectorBucketOf, vectorBucketScope } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; import type { CompressedObservation } from "../src/types.js"; -const BM25_SCOPE = "mem:index:bm25"; -const BM25_LEGACY_KEY = "data"; -const BM25_MANIFEST_KEY = "data:manifest"; -const VECTOR_LEGACY_KEY = "vectors"; -const VECTOR_MANIFEST_KEY = "vectors:manifest"; - -type TestIndexShardManifest = { - v: 1; - generation?: string; - shards: Array<{ scope: string; key: string; chars: number }>; - chars: number; -}; +const INDEX_SCOPE = "mem:index:bm25"; +const META_KEY = "vectors:meta"; function mockKV() { const store = new Map>(); + const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = []; return { - get: async (scope: string, key: string): Promise => { - return (store.get(scope)?.get(key) as T) ?? null; - }, + store, + ops, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, set: async (scope: string, key: string, data: T): Promise => { + ops.push({ op: "set", scope, key }); if (!store.has(scope)) store.set(scope, new Map()); store.get(scope)!.set(key, data); return data; }, delete: async (scope: string, key: string): Promise => { + ops.push({ op: "delete", scope, key }); store.get(scope)?.delete(key); }, list: async (scope: string): Promise => { - const entries = store.get(scope); - return entries ? (Array.from(entries.values()) as T[]) : []; + ops.push({ op: "list", scope }); + return Array.from(store.get(scope)?.values() ?? []) as T[]; }, }; } type MockKV = ReturnType; -function makeObs( - overrides: Partial = {}, -): CompressedObservation { +function vec(values: number[]): Float32Array { + return new Float32Array(values); +} + +function vectorWith(entries: Array<[string, number[]]>): VectorIndex { + const index = new VectorIndex(); + for (const [id, values] of entries) index.add(id, `ses_${id}`, vec(values)); + return index; +} + +function bucketSets(kv: MockKV): Array<{ scope: string; key?: string }> { + return kv.ops.filter((o) => o.op === "set" && o.scope.startsWith(`${INDEX_SCOPE}:vec:`)); +} + +function obs(id: string): CompressedObservation { return { - id: "obs_1", + id, sessionId: "ses_1", timestamp: new Date().toISOString(), type: "file_edit", - title: "Edit auth middleware", - subtitle: "JWT validation", - facts: ["Added token check"], - narrative: "Modified the auth middleware to validate JWT tokens", - concepts: ["authentication", "jwt"], - files: ["src/middleware/auth.ts"], - importance: 7, - ...overrides, + title: `title ${id}`, + facts: [], + narrative: `narrative ${id}`, + concepts: [], + files: [], + importance: 5, }; } -function makeBm25(id: string, title: string): SearchIndex { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id, title, narrative: `${title} narrative` })); - return bm25; +async function writeLegacyVectorSnapshot( + kv: MockKV, + vector: VectorIndex, + options: { generation?: string; monolithic?: boolean; shardChars?: number } = {}, +): Promise { + const serialized = vector.serialize(); + if (options.monolithic) { + await kv.set(INDEX_SCOPE, "vectors", serialized); + return; + } + const generation = options.generation ?? "idx_mfabcd12_aaaaaaaaaaaa"; + const size = options.shardChars ?? 40; + const shards: Array<{ scope: string; key: string; chars: number }> = []; + for (let offset = 0, i = 0; offset < serialized.length; offset += size, i++) { + const scope = `${INDEX_SCOPE}:vectors:${generation}:${String(i).padStart(5, "0")}`; + const chunk = serialized.slice(offset, offset + size); + await kv.set(scope, "data", chunk); + shards.push({ scope, key: "data", chars: chunk.length }); + } + await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, generation, shards, chars: serialized.length }); } -function makeVector(id = "obs_1"): VectorIndex { - const vector = new VectorIndex(); - vector.add(id, "ses_1", new Float32Array([0.1, 0.2, 0.3])); - return vector; +async function writeLegacyBm25Snapshot(kv: MockKV): Promise { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_legacy")); + const serialized = bm25.serialize(); + const scope = `${INDEX_SCOPE}:bm25:idx_mfabcd12_bbbbbbbbbbbb:00000`; + await kv.set(scope, "data", serialized); + await kv.set(INDEX_SCOPE, "data:manifest", { + v: 1, + generation: "idx_mfabcd12_bbbbbbbbbbbb", + shards: [{ scope, key: "data", chars: serialized.length }], + chars: serialized.length, + }); } -async function getBm25Manifest(kv: MockKV): Promise { - const manifest = await kv.get( - BM25_SCOPE, - BM25_MANIFEST_KEY, - ); - expect(manifest).not.toBeNull(); - return manifest!; +function expectSameVectors(actual: VectorIndex | null, expected: VectorIndex): void { + expect(actual).not.toBeNull(); + expect(actual!.size).toBe(expected.size); + for (const [id, entry] of expected.entries()) { + const loaded = actual!.get(id); + expect(loaded?.sessionId).toBe(entry.sessionId); + expect(Array.from(loaded!.embedding)).toEqual(Array.from(entry.embedding)); + } } -describe("IndexPersistence", () => { - let kv: ReturnType; +describe("IndexPersistence bucketed vector storage", () => { + let kv: MockKV; beforeEach(() => { - vi.useFakeTimers(); kv = mockKV(); }); - afterEach(() => { - vi.useRealTimers(); - }); - - it("saves and loads BM25 index round-trip", async () => { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); + it("round-trips vectors through fixed bucket scopes", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ["obs_c", [0.7, 0.8, 0.9]], + ]); + await new IndexPersistence(kv as never, vector, { buckets: 16 }).save(); - const persistence = new IndexPersistence(kv as never, bm25, null); - await persistence.save(); + for (const [id] of vector.entries()) { + expect(kv.store.get(vectorBucketScope(vectorBucketOf(id, 16)))?.has(id)).toBe(true); + } + const meta = await kv.get<{ v: number; buckets: number; count: number }>(INDEX_SCOPE, META_KEY); + expect(meta).toMatchObject({ v: 2, buckets: 16, count: 3 }); - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.size).toBe(1); - const results = loaded.bm25!.search("auth"); - expect(results.length).toBe(1); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(loaded.state).toBe("buckets"); + expectSameVectors(loaded.vector, vector); + expect(loaded.vector!.pendingChanges).toBe(0); }); - it("saves BM25 index shards outside the BM25 metadata scope", async () => { - const bm25 = new SearchIndex(); - bm25.add( - makeObs({ - id: "obs_1", - title: "auth handler ".repeat(40), - narrative: "JWT middleware validation ".repeat(40), - }), - ); - - const persistence = new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_bm25", - }); + it("writes only the bucket entry of a single added vector", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); await persistence.save(); + kv.ops.length = 0; - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_bm25"); - expect(manifest.shards.length).toBeGreaterThan(1); - expect(manifest.shards[0].scope).toContain(":gen_bm25:"); - await expect(kv.get(BM25_SCOPE, BM25_LEGACY_KEY)).resolves.toBeNull(); - await expect( - kv.get(manifest.shards[0].scope, manifest.shards[0].key), - ).resolves.toEqual(expect.any(String)); - - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("auth").length).toBe(1); - }); - - it("loads legacy monolithic BM25 and vector snapshots", async () => { - const bm25 = makeBm25("obs_1", "legacy auth handler"); - const vector = makeVector("obs_1"); - await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, bm25.serialize()); - await kv.set(BM25_SCOPE, VECTOR_LEGACY_KEY, vector.serialize()); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("legacy").length).toBe(1); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); - }); - - it("fails closed instead of falling back when manifest reads fail", async () => { - const legacy = makeBm25("obs_legacy", "legacy stale snapshot"); - await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, legacy.serialize()); - const failingKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - throw new Error("manifest backend unavailable"); - } - return kv.get(scope, key); - }), - }; - - const loaded = await new IndexPersistence( - failingKv as never, - new SearchIndex(), - null, - ).load(); + vector.add("obs_new", "ses_new", vec([1, 0, 0])); + await persistence.save(); - expect(loaded.bm25).toBeNull(); + const writes = bucketSets(kv); + expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(vectorBucketOf("obs_new", 16)), key: "obs_new" }]); }); - it("fails closed when legacy snapshot reads fail", async () => { - const failingKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === BM25_SCOPE && key === BM25_LEGACY_KEY) { - throw new Error("legacy backend unavailable"); - } - return kv.get(scope, key); - }), - }; - - const loaded = await new IndexPersistence( - failingKv as never, - new SearchIndex(), - null, - ).load(); + it("does not write when nothing changed", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + await persistence.save(); + kv.ops.length = 0; - expect(loaded.bm25).toBeNull(); - }); + await persistence.save(); - it("loads sharded manifests that omit optional generation metadata", async () => { - const bm25 = makeBm25("obs_1", "deterministic shard auth"); - const serialized = bm25.serialize(); - const chunks = [serialized.slice(0, 50), serialized.slice(50)]; - await kv.set("mem:index:bm25:bm25:00000", "data", chunks[0]); - await kv.set("mem:index:bm25:bm25:00001", "data", chunks[1]); - await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { - v: 1, - chars: serialized.length, - shards: [ - { - scope: "mem:index:bm25:bm25:00000", - key: "data", - chars: chunks[0].length, - }, - { - scope: "mem:index:bm25:bm25:00001", - key: "data", - chars: chunks[1].length, - }, - ], - }); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("deterministic").length).toBe(1); + expect(kv.ops.filter((o) => o.op !== "list")).toEqual([]); }); - it("saves and loads vector index round-trip", async () => { - const bm25 = new SearchIndex(); - const vector = makeVector(); + it("removes deleted vectors from their bucket", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + await persistence.save(); - const persistence = new IndexPersistence(kv as never, bm25, vector); + vector.remove("obs_a"); await persistence.save(); - const loaded = await persistence.load(); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); + expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(vectorBucketOf("obs_a", 16)), key: "obs_a" }); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(loaded.vector!.has("obs_a")).toBe(false); + expect(loaded.vector!.has("obs_b")).toBe(true); }); - it("saves vector index shards outside the BM25 scope", async () => { - const bm25 = new SearchIndex(); - const vector = new VectorIndex(); - vector.add( - "obs_1", - "ses_1", - new Float32Array(Array.from({ length: 32 }, (_, i) => i)), - ); - - const persistence = new IndexPersistence(kv as never, bm25, vector, { - shardChars: 40, - createGeneration: () => "gen_vector", - }); + it("clearing the index deletes every persisted vector", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); await persistence.save(); - const manifest = await kv.get( - BM25_SCOPE, - VECTOR_MANIFEST_KEY, - ); - expect(manifest).not.toBeNull(); - expect(manifest!.generation).toBe("gen_vector"); - expect(manifest!.shards.length).toBeGreaterThan(1); - expect(manifest!.shards[0].scope).toContain(":gen_vector:"); - await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull(); - await expect( - kv.get(manifest!.shards[0].scope, manifest!.shards[0].key), - ).resolves.toEqual(expect.any(String)); - - const loaded = await persistence.load(); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); - }); + vector.clear(); + vector.add("obs_c", "ses_c", vec([0, 1, 0])); + await persistence.save(); - it("persists empty vector snapshots so cleared vectors do not reload", async () => { - const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); - const previousVector = makeVector("obs_old"); - await new IndexPersistence(kv as never, previousBm25, previousVector, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - - const nextBm25 = makeBm25("obs_new", "bravo new snapshot"); - const emptyVector = new VectorIndex(); - await new IndexPersistence(kv as never, nextBm25, emptyVector, { - shardChars: 80, - createGeneration: () => "gen_empty", - }).save(); - - const vectorManifest = await kv.get( - BM25_SCOPE, - VECTOR_MANIFEST_KEY, - ); - expect(vectorManifest).not.toBeNull(); - expect(vectorManifest!.generation).toBe("gen_empty"); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(0); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect([...loaded.vector!.entries()].map(([id]) => id)).toEqual(["obs_c"]); }); - it("avoids one oversized state::set string payload for persisted indexes", async () => { - const maxStringPayloadChars = 80; - const bm25 = new SearchIndex(); - bm25.add( - makeObs({ - id: "obs_1", - title: "large persisted snapshot ".repeat(40), - narrative: "oversized state set reproduction ".repeat(40), - }), - ); - const vector = new VectorIndex(); - vector.add( - "obs_1", - "ses_1", - new Float32Array(Array.from({ length: 64 }, (_, i) => i / 10)), - ); - const guardedKv = { + it("keeps failed writes pending and retries them on the next save", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + let fail = true; + const flakyKv = { ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if ( - typeof data === "string" && - data.length > maxStringPayloadChars - ) { - throw new Error(`oversized state::set payload: ${scope}/${key}`); - } + set: async (scope: string, key: string, data: T): Promise => { + if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("state::set timed out"); return kv.set(scope, key, data); - }), + }, }; + const persistence = new IndexPersistence(flakyKv as never, vector, { buckets: 16 }); - await new IndexPersistence(guardedKv as never, bm25, vector, { - shardChars: maxStringPayloadChars, - createGeneration: () => "gen_payload_limit", - }).save(); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("oversized").length).toBe(1); - expect(loaded.vector!.size).toBe(1); - }); + await expect(persistence.save()).resolves.toBeUndefined(); + const failed = persistence.status(); + expect(failed.vector?.lastError).toBe("1 of 1 vector writes failed: state::set timed out"); + expect(failed.pendingChanges).toBe(1); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); - it("falls back to the default shard size for fractional values below one", async () => { - const bm25 = makeBm25("obs_fraction", "fractional shard config"); - let newShardWrites = 0; - const guardedKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope.includes(":gen_fraction:")) { - newShardWrites += 1; - if (newShardWrites > 3) { - throw new Error("fractional shard size caused zero-width shards"); - } - } - return kv.set(scope, key, data); - }), - }; + fail = false; + await persistence.save(); + expect(persistence.status().vector?.lastError).toBeNull(); + expect(persistence.status().pendingChanges).toBe(0); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expectSameVectors(loaded.vector, vector); + }); - await new IndexPersistence(guardedKv as never, bm25, null, { - shardChars: 0.5, - createGeneration: () => "gen_fraction", - }).save(); + it("moves vectors when the bucket count changes", async () => { + const vector = vectorWith( + Array.from({ length: 40 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + ); + await new IndexPersistence(kv as never, vector, { buckets: 4 }).save(); + + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 8 }).load(); + expectSameVectors(loaded.vector, vector); + for (const [id] of vector.entries()) { + const oldScope = vectorBucketScope(vectorBucketOf(id, 4)); + const newScope = vectorBucketScope(vectorBucketOf(id, 8)); + expect(kv.store.get(newScope)?.has(id)).toBe(true); + if (oldScope !== newScope) expect(kv.store.get(oldScope)?.has(id) ?? false).toBe(false); + } - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_fraction"); - expect(manifest.shards.length).toBe(1); - expect(newShardWrites).toBe(1); + const again = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 8 }).load(); + expect(again.vector!.size).toBe(40); }); - it("keeps the previous generation when a shard write fails before manifest commit", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); - - let newShardWrites = 0; + it("reports storage as unavailable when the metadata read fails", async () => { const failingKv = { ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope.includes(":gen_new:")) { - newShardWrites += 1; - if (newShardWrites === 2) throw new Error("shard write failed"); - } - return kv.set(scope, key, data); - }), + get: async () => { + throw new Error("engine down"); + }, }; - - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null }); }); - it("keeps the previous generation when manifest set rejects before commit", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); - - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - throw new Error("manifest write failed"); - } - return kv.set(scope, key, data); - }), - }; + it("treats a missing store as empty, including adapters that return undefined", async () => { + const undefinedKv = { ...kv, get: async () => undefined }; + const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); + }); +}); - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); +describe("IndexPersistence migration from the single-string format", () => { + let kv: MockKV; - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + beforeEach(() => { + kv = mockKV(); }); - it("keeps a generation loadable when manifest set commits before rejecting", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - await kv.set(scope, key, data); - throw new Error("manifest write timed out after commit"); - } - return kv.set(scope, key, data); - }), - }; + it("migrates a sharded vector snapshot into buckets and removes the old shards", async () => { + const legacy = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ["obs_c", [0.7, 0.8, 0.9]], + ]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); + await writeLegacyBm25Snapshot(kv); + + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + + expect(loaded.state).toBe("migrated"); + expect(loaded.savedAt).toBe(new Date(parseInt("mfabcd12", 36)).toISOString()); + expectSameVectors(loaded.vector, legacy); + expect(loaded.vector!.pendingChanges).toBe(0); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull(); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull(); + for (const [scope, entries] of kv.store) { + if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0); + } - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_new"); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toEqual(expect.any(String)); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); + const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(reloaded.state).toBe("buckets"); + expectSameVectors(reloaded.vector, legacy); }); - it("deletes a shard that committed before set rejected", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); + it("migrates a monolithic vector snapshot", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true }); - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === "mem:index:bm25:bm25:gen_new:00000") { - await kv.set(scope, key, data); - throw new Error("state::set timed out after commit"); - } - return kv.set(scope, key, data); - }), - }; + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + expect(loaded.state).toBe("migrated"); + expectSameVectors(loaded.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors")).toBeNull(); }); - it("loads the new generation even when old generation cleanup fails", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); + it("leaves an unreadable legacy snapshot in place and writes nothing", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy); + const manifest = await kv.get<{ shards: Array<{ scope: string; key: string }> }>(INDEX_SCOPE, "vectors:manifest"); + await kv.delete(manifest!.shards[0].scope, manifest!.shards[0].key); + kv.ops.length = 0; - const cleanupKv = { - ...kv, - delete: vi.fn(async () => { - throw new Error("cleanup failed"); - }), - }; - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(cleanupKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_new"); - expect(cleanupKv.delete).toHaveBeenCalled(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.bm25!.search("alpha").length).toBe(0); - }); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); - it("keeps the previous vector generation when vector save fails after BM25 publish", async () => { - const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); - const previousVector = makeVector("obs_old"); - await new IndexPersistence(kv as never, previousBm25, previousVector, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); + expect(loaded.state).toBe("unavailable"); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); + expect(kv.ops.filter((o) => o.op === "set" || o.op === "delete")).toEqual([]); + }); - const failingKv = { + it("keeps the legacy snapshot when migration writes fail and finishes on a later save", async () => { + const legacy = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + await writeLegacyVectorSnapshot(kv, legacy); + let fail = true; + const flakyKv = { ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { - throw new Error("vector manifest write failed"); - } + set: async (scope: string, key: string, data: T): Promise => { + if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("write failed"); return kv.set(scope, key, data); - }), + }, }; - const nextBm25 = makeBm25("obs_new", "bravo new snapshot"); - const nextVector = new VectorIndex(); - nextVector.add("obs_new", "ses_1", new Float32Array([0.4, 0.5, 0.6])); - - await new IndexPersistence(failingKv as never, nextBm25, nextVector, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - await expect( - kv.get("mem:index:bm25:vectors:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.vector!.size).toBe(1); - expect( - loaded.vector!.search(new Float32Array([0.1, 0.2, 0.3]))[0]?.obsId, - ).toBe("obs_old"); - }); - - it("fails closed when a manifest shard is missing", async () => { - const bm25 = makeBm25("obs_1", "alpha sharded snapshot"); - await new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_missing", - }).save(); - const manifest = await getBm25Manifest(kv); - await kv.delete(manifest.shards[0].scope, manifest.shards[0].key); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).toBeNull(); - }); - it("fails closed when a manifest shard length mismatches", async () => { - const bm25 = makeBm25("obs_1", "alpha sharded snapshot"); - await new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_mismatch", - }).save(); - const manifest = await getBm25Manifest(kv); - const firstShard = manifest.shards[0]; - const chunk = await kv.get(firstShard.scope, firstShard.key); - await kv.set(firstShard.scope, firstShard.key, `${chunk}x`); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).toBeNull(); - }); - - it("fails closed before reading invalid shard descriptors", async () => { - await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { - v: 1, - chars: 10, - shards: [{ scope: "", key: "data", chars: 10 }], - }); - const guardedKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === "") { - throw new Error("invalid shard descriptor was read"); - } - return kv.get(scope, key); - }), - }; + const live = new VectorIndex(); + const persistence = new IndexPersistence(flakyKv as never, live, { buckets: 16 }); + const loaded = await persistence.load(); + expect(loaded.state).toBe("unavailable"); + expectSameVectors(loaded.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); - const loaded = await new IndexPersistence( - guardedKv as never, - new SearchIndex(), - null, - ).load(); + live.restoreFrom(loaded.vector!); + fail = false; + await persistence.save(); - expect(loaded.bm25).toBeNull(); - expect(guardedKv.get).not.toHaveBeenCalledWith("", "data"); + const next = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(next.state).toBe("buckets"); + expectSameVectors(next.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull(); }); - it("scheduleSave debounces multiple calls", async () => { - const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 }); + it("without vector search, removes only the old BM25 snapshot", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy); + await writeLegacyBm25Snapshot(kv); - persistence.scheduleSave(); - persistence.scheduleSave(); - persistence.scheduleSave(); - - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toBeNull(); - - vi.advanceTimersByTime(5000); - await vi.runAllTimersAsync(); + const loaded = await new IndexPersistence(kv as never, null, { buckets: 16 }).load(); - const saved = await kv.get(BM25_SCOPE, BM25_MANIFEST_KEY); - expect(saved).not.toBeNull(); + expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull(); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); }); +}); - it("stop clears the pending timer", async () => { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(kv as never, bm25, null); - - persistence.scheduleSave(); - persistence.stop(); +describe("IndexPersistence scheduling", () => { + let kv: MockKV; - vi.advanceTimersByTime(10000); - const saved = await kv.get(BM25_SCOPE, BM25_MANIFEST_KEY); - expect(saved).toBeNull(); + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); }); - it("returns null indexes when nothing has been saved", async () => { - const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null); - - const loaded = await persistence.load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector).toBeNull(); + afterEach(() => { + vi.useRealTimers(); }); - it("scheduled save swallows kv.set rejection without unhandledRejection (#204)", async () => { + it("scheduled saves never raise unhandled rejections", async () => { const failingKv = { - ...mockKV(), + ...kv, set: vi.fn(async () => { - const err = new Error( - "TIMEOUT: invocation timed out after 30000ms", - ) as Error & { code?: string; function_id?: string }; + const err = new Error("TIMEOUT: invocation timed out after 30000ms") as Error & { code?: string }; err.code = "TIMEOUT"; - err.function_id = "state::set"; throw err; }), }; - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); - + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, buckets: 16 }); let unhandled = false; const onUnhandled = () => { unhandled = true; }; process.on("unhandledRejection", onUnhandled); - try { persistence.scheduleSave(); - vi.advanceTimersByTime(5000); - await vi.runAllTimersAsync(); - // give microtasks a chance to flush - await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5000); expect(failingKv.set).toHaveBeenCalled(); expect(unhandled).toBe(false); } finally { @@ -742,143 +400,12 @@ describe("IndexPersistence", () => { } }); - it("save() does not throw when kv.set rejects (#204)", async () => { - const failingKv = { - ...mockKV(), - set: vi.fn(async () => { - throw new Error("TIMEOUT"); - }), - }; - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); - - await expect(persistence.save()).resolves.toBeUndefined(); - }); - - // #797: first run after upgrading to 0.9.25 crashed with - // 'TypeError: Cannot read properties of undefined (reading "v")' - // because some iii-state adapters return `undefined` (not `null`) - // for a missing key. The load path's `value !== null` check passed - // undefined to loadManifestData, which then read `undefined.v`. - it("load() returns null instead of crashing when kv.get returns undefined for the manifest (#797)", async () => { - const undefinedKv = { - ...mockKV(), - get: vi.fn(async () => undefined), - }; - const persistence = new IndexPersistence( - undefinedKv as never, - new SearchIndex(), - null, - ); - - const loaded = await persistence.load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector).toBeNull(); - }); - - it("load() does not crash when a manifest row value is the wrong shape (#797)", async () => { - const wrongShapeKv = { - ...mockKV(), - get: vi.fn(async () => "not-a-manifest"), - }; - const persistence = new IndexPersistence( - wrongShapeKv as never, - new SearchIndex(), - null, - ); - - await expect(persistence.load()).resolves.toBeDefined(); - }); -}); - -describe("index_persist audit gating", () => { - let kv: ReturnType; - let previousFlag: string | undefined; - - beforeEach(() => { - // AGENTMEMORY_* variables are documented as living in - // ~/.agentmemory/.env, so a developer running the suite on a - // configured machine can inherit this one. Clear it going in and put - // whatever was there back on the way out. - previousFlag = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - vi.useFakeTimers(); - kv = mockKV(); - }); - - afterEach(() => { - vi.useRealTimers(); - if (previousFlag === undefined) { - delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - } else { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previousFlag; - } - }); - - async function indexPersistEntries(): Promise> { - const entries = await kv.list<{ operation: string }>("mem:audit"); - return entries.filter((entry) => entry.operation === "index_persist"); - } - - it("writes no index_persist audit entries by default", async () => { - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect(await indexPersistEntries()).toEqual([]); - }); - - it.each(["1", " 1 ", "true", "TRUE", " true "])( - "writes index_persist audit entries when set to %j", - async (value) => { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect((await indexPersistEntries()).length).toBeGreaterThan(0); - }, - ); - - // Anything that is not an affirmative stays off. "0" and "false" are the - // ones an operator is likely to reach for to disable it, and they must - // not read as "present, therefore enabled". - it.each(["0", "false", "yes", "", " "])( - "keeps auditing off when set to %j", - async (value) => { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect(await indexPersistEntries()).toEqual([]); - }, - ); - - it("still persists the index when auditing is off", async () => { - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.size).toBe(1); + it("stop clears the pending timer", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, buckets: 16 }); + persistence.scheduleSave(); + persistence.stop(); + await vi.advanceTimersByTimeAsync(10_000); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); }); }); From 51f88a5fc89dec949e9988171d9276a3fb000f65 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:33:55 +0530 Subject: [PATCH 3/7] chore: credit co-authors The per-leg save failure handling and the single save queue build on the fix in #1256. The problem was measured and reported in discussion #1358. Co-authored-by: Omar Gerardo Co-authored-by: Vladimir <7998636+MarvinFS@users.noreply.github.com> From 49cf1595e6de64535becd7970958d23e19b98427 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:33:55 +0530 Subject: [PATCH 4/7] chore: credit co-authors The string-limit and whole-generation rewrite analysis comes from discussion #1358; the reclaim work in #1256 mapped the generation lifecycle this replaces. Co-authored-by: Omar Gerardo Co-authored-by: Vladimir <7998636+MarvinFS@users.noreply.github.com> From 14cb1a69288309db632659b30631ff30d0df98c0 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:42:39 +0530 Subject: [PATCH 5/7] fix(index-persistence): snapshot dirtyEpoch per leg before serializing runSave() captured one dirtyEpoch before the BM25 leg's write and reused it for the vector leg's dirty check. When scheduleSave() ran while the BM25 write was in flight, the vector leg would serialize the current (already up to date) state but still compare against the stale epoch, leaving vector.dirtySince set even though the just-written snapshot already reflected the change. status() would report the vector leg as dirty until an unrelated follow-up save cleared it. saveLeg() now snapshots dirtyEpoch itself, immediately before invoking the leg's own serialize+write callback, so each leg judges its dirty state against the epoch as of its own write rather than one captured before an earlier leg ran. Added a regression test that gates the BM25 manifest write, triggers a scheduleSave() while it is held, and asserts the vector leg's dirtySince clears once its own write completes. --- src/state/index-persistence.ts | 8 +++--- test/index-persistence-throttle.test.ts | 33 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index a77d66ed30..28a24d5a1c 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -218,17 +218,16 @@ export class IndexPersistence { } private async runSave(): Promise { - const epoch = this.dirtyEpoch; this.markedDuringRunAt = null; this.lastSaveAt = this.now(); - await this.saveLeg("bm25", epoch, async () => { + await this.saveLeg("bm25", async () => { const serialized = this.bm25.serialize(); this.legs.bm25.serializedChars = serialized.length; await this.saveBm25Index(serialized); }); const vector = this.vector; if (vector) { - await this.saveLeg("vector", epoch, async () => { + await this.saveLeg("vector", async () => { const serialized = vector.serialize(); this.legs.vector.serializedChars = serialized.length; await this.saveVectorIndex(serialized); @@ -236,8 +235,9 @@ export class IndexPersistence { } } - private async saveLeg(leg: IndexLeg, epoch: number, run: () => Promise): Promise { + private async saveLeg(leg: IndexLeg, run: () => Promise): Promise { const status = this.legs[leg]; + const epoch = this.dirtyEpoch; try { await run(); status.lastSavedAt = new Date(this.now()).toISOString(); diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts index 8a013fa662..54eff133b8 100644 --- a/test/index-persistence-throttle.test.ts +++ b/test/index-persistence-throttle.test.ts @@ -189,6 +189,39 @@ describe("IndexPersistence save throttling", () => { expect(persistence.status().bm25.dirtySince).toBeNull(); }); + it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let heldBm25Manifest = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) { + heldBm25Manifest = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + const status = persistence.status(); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + }); + it("stop prevents later scheduled saves", async () => { const bm25 = new SearchIndex(); bm25.add(obs("obs_1", "alpha")); From 400c68ce86fb186076478b125f8841ac116de107 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 13:25:00 +0530 Subject: [PATCH 6/7] fix(index): close vector-index safety gaps from PR review Guard rebuildKeywordIndex's sessions listing with a retried helper instead of a bare kv.list call; a failure now reports an incomplete BM25 rebuild on status instead of throwing. mem::search's cold-start path now calls rebuildKeywordIndex instead of rebuildIndex, so a query against an empty index can never clear or re-embed the persisted vector index; only an explicit dimension mismatch at boot still drops vectors. Gate the vector backfill so a whole-store embed never starts silently: AGENTMEMORY_VECTOR_BACKFILL=all is required when there is no prior save to catch up from, backfills are capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX (default 500), the pending count is reported on /agentmemory/status, and backfillVectors now saves progress every few batches so a restart does not repeat finished embedding work. On load, compare the vector count actually read against the count the last save recorded; a shortfall is reported on status and falls back to a bounded backfill for the missing ids instead of trusting the saved timestamp. Meta is still written only after every bucket write of a save succeeds. Replace hash-mod-N vector bucketing with fill-and-roll: every vector's bucket is tracked in memory, built by reading all known buckets at load; new vectors fill the current open bucket up to AGENTMEMORY_VECTOR_BUCKET_SIZE (default 500) before a new bucket opens, and updates/deletes go straight to a vector's existing bucket. A save under load now touches only the buckets it actually changed instead of rewriting the whole store, and a bucket-size change no longer requires moving existing data. Migration from the pre-PR sharded/monolithic format is unchanged; the hash-bucket layout from this PR's earlier, unreleased commit is dropped rather than migrated. Tests: sessions-listing failure keeps vectors untouched and marks the rebuild incomplete; whole-store backfill opt-in and per-boot cap; partial bucket set detected against the saved count; a save spread over several waves of adds touches only the open bucket; deletes touch only the owning bucket; migration from the old sharded format still passes. --- .env.example | 4 +- plugin/skills/agentmemory-config/REFERENCE.md | 6 +- src/config.ts | 19 ++- src/functions/search.ts | 93 +++++++++-- src/functions/status.ts | 31 ++++ src/index.ts | 16 +- src/state/index-persistence.ts | 148 ++++++++++-------- src/triggers/api.ts | 11 +- test/index-keyword-rebuild.test.ts | 77 +++++++-- test/index-persistence-throttle.test.ts | 22 ++- test/index-persistence.test.ts | 143 ++++++++++++----- test/search.test.ts | 17 ++ test/status.test.ts | 24 +++ 13 files changed, 468 insertions(+), 143 deletions(-) diff --git a/.env.example b/.env.example index f42dd7f272..60d201e56a 100644 --- a/.env.example +++ b/.env.example @@ -130,7 +130,9 @@ # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). # AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. -# AGENTMEMORY_VECTOR_BUCKETS=256 # Number of state scopes the vector index is spread across; a save rewrites only the buckets it touched. +# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed. +# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save. +# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index abbf39e9f7..44e300d1df 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 45 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -46,7 +46,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_TOOLS` - `AGENTMEMORY_URL` - `AGENTMEMORY_USE_DOCKER` -- `AGENTMEMORY_VECTOR_BUCKETS` +- `AGENTMEMORY_VECTOR_BACKFILL` +- `AGENTMEMORY_VECTOR_BACKFILL_MAX` +- `AGENTMEMORY_VECTOR_BUCKET_SIZE` - `AGENTMEMORY_VERBOSE` - `AGENTMEMORY_VIEWER_HOST` - `AGENTMEMORY_VIEWER_URL` diff --git a/src/config.ts b/src/config.ts index ae9fd19f29..a7986f9604 100644 --- a/src/config.ts +++ b/src/config.ts @@ -477,11 +477,22 @@ export function getIndexSaveIntervalMs(): number { return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; } -export const VECTOR_BUCKETS_DEFAULT = 256; +export const VECTOR_BUCKET_SIZE_DEFAULT = 500; -export function getVectorBucketCount(): number { - const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKETS"], VECTOR_BUCKETS_DEFAULT); - return raw > 0 && raw <= 65_536 ? raw : VECTOR_BUCKETS_DEFAULT; +export function getVectorBucketSize(): number { + const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKET_SIZE"], VECTOR_BUCKET_SIZE_DEFAULT); + return raw > 0 ? raw : VECTOR_BUCKET_SIZE_DEFAULT; +} + +export const VECTOR_BACKFILL_MAX_DEFAULT = 500; + +export function getVectorBackfillMax(): number { + const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL_MAX"], VECTOR_BACKFILL_MAX_DEFAULT); + return raw > 0 ? raw : VECTOR_BACKFILL_MAX_DEFAULT; +} + +export function isVectorBackfillAllEnabled(): boolean { + return getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL"] === "all"; } export function isStandaloneMcp(): boolean { diff --git a/src/functions/search.ts b/src/functions/search.ts index dc834502c7..2d2f414785 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -9,7 +9,12 @@ import type { EmbeddingProvider } from '../types.js' import { memoryToObservation } from '../state/memory-utils.js' import { recordAccessBatch } from './access-tracker.js' import { logger } from "../logger.js"; -import { getAgentId, isAgentScopeIsolated } from "../config.js"; +import { + getAgentId, + getVectorBackfillMax, + isAgentScopeIsolated, + isVectorBackfillAllEnabled, +} from "../config.js"; let index: SearchIndex | null = null let vectorIndex: VectorIndex | null = null @@ -45,6 +50,19 @@ export function isMemoryIndexReady(): boolean { return memoryIndexReady } +let bm25RebuildIncomplete = false +export function isBm25RebuildIncomplete(): boolean { + return bm25RebuildIncomplete +} + +let pendingVectorBackfill = 0 +export function getPendingVectorBackfillCount(): number { + return pendingVectorBackfill +} +export function setPendingVectorBackfillCount(n: number): void { + pendingVectorBackfill = Math.max(0, n) +} + export function getSearchIndex(): SearchIndex { if (!index) index = new SearchIndex() return index @@ -406,17 +424,49 @@ export type VectorBackfillJob = { context: { kind: "memory" | "observation" | "synthetic"; logId: string } } +const SESSIONS_LIST_RETRY_ATTEMPTS = 3 +const SESSIONS_LIST_RETRY_BASE_MS = 250 + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function listSessionsWithRetry(kv: StateKV): Promise { + let lastErr: unknown + for (let attempt = 0; attempt < SESSIONS_LIST_RETRY_ATTEMPTS; attempt++) { + try { + return await kv.list(KV.sessions) + } catch (err) { + lastErr = err + if (attempt < SESSIONS_LIST_RETRY_ATTEMPTS - 1) { + await delay(SESSIONS_LIST_RETRY_BASE_MS * (attempt + 1)) + } + } + } + logger.warn('rebuildKeywordIndex: failed to load sessions after retries', { + attempts: SESSIONS_LIST_RETRY_ATTEMPTS, + error: lastErr instanceof Error ? lastErr.message : String(lastErr), + }) + return null +} + export async function rebuildKeywordIndex( kv: StateKV, vectorBackfillSince?: string | null, -): Promise<{ documents: number; vectorJobs: VectorBackfillJob[] }> { +): Promise<{ documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number }> { const idx = getSearchIndex() idx.clear() memoryIndexReady = false + bm25RebuildIncomplete = false const vi = vectorIndex - const backfill = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined + const backfillEligible = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined + const wholeStoreBackfill = vectorBackfillSince === null + const backfillGated = backfillEligible && wholeStoreBackfill && !isVectorBackfillAllEnabled() + const backfillActive = backfillEligible && !backfillGated const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN + const backfillCap = backfillActive ? getVectorBackfillMax() : 0 const vectorJobs: VectorBackfillJob[] = [] + let fullBackfillPending = 0 const consider = ( id: string, sessionId: string, @@ -424,8 +474,13 @@ export async function rebuildKeywordIndex( timestamp: string | undefined, kind: "memory" | "observation", ): void => { - if (!backfill || vi?.has(id)) return + if (!backfillEligible || vi?.has(id)) return if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return + if (backfillGated) { + fullBackfillPending++ + return + } + if (!backfillActive || vectorJobs.length >= backfillCap) return vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } }) } @@ -447,7 +502,12 @@ export async function rebuildKeywordIndex( }) } - const sessions = await kv.list(KV.sessions) + const sessions = await listSessionsWithRetry(kv) + if (sessions === null) { + bm25RebuildIncomplete = true + if (memoriesLoaded) memoryIndexReady = true + return { documents, vectorJobs, fullBackfillPending } + } const failedSessions: string[] = [] for (let batch = 0; batch < sessions.length; batch += 10) { const chunk = sessions.slice(batch, batch + 10) @@ -469,20 +529,31 @@ export async function rebuildKeywordIndex( } } if (failedSessions.length > 0) { + bm25RebuildIncomplete = true logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions }) } if (memoriesLoaded) memoryIndexReady = true - return { documents, vectorJobs } + return { documents, vectorJobs, fullBackfillPending } } +const BACKFILL_SAVE_EVERY_BATCHES = 10 + export async function backfillVectors(jobs: VectorBackfillJob[]): Promise { const batchSize = getRebuildEmbedBatchSize() let added = 0 + let batchesSinceSave = 0 + setPendingVectorBackfillCount(jobs.length) for (let offset = 0; offset < jobs.length; offset += batchSize) { const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize)) added += ok + setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize)) + batchesSinceSave++ + if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) { + await flushIndexSave() + batchesSinceSave = 0 + } } - if (added > 0) scheduleIndexSave() + if (added > 0) await flushIndexSave() return added } @@ -567,10 +638,10 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { // Share one rebuild across concurrent cold-start queries so they // don't each walk the whole corpus and saturate the pool. if (!rebuildPromise) { - rebuildPromise = rebuildIndex(kv) - .then((count) => { - logger.info('Search index rebuilt', { entries: count }) - return count + rebuildPromise = rebuildKeywordIndex(kv) + .then((result) => { + logger.info('Search index rebuilt', { entries: result.documents }) + return result.documents }) .catch((err) => { logger.warn('Index rebuild failed', { diff --git a/src/functions/status.ts b/src/functions/status.ts index db1d49d789..c6577c44e6 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -57,6 +57,8 @@ export interface StatusInputs { observationsIndexed: number; missingObservations: number | null; sessions: number | null; + bm25Incomplete: boolean; + pendingVectorBackfill: number; }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; @@ -172,6 +174,24 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } + if (input.index.bm25Incomplete) { + problems.push({ + level: "error", + code: "bm25-rebuild-incomplete", + message: "The keyword index could not load every session at boot, so some observations are missing from search until the next successful rebuild.", + fix: "Check the server log for the session listing failure. Restarting or the next cold-start search retries automatically.", + }); + } + + if (input.index.pendingVectorBackfill > 0) { + problems.push({ + level: "info", + code: "index-vector-backfill-pending", + message: `${input.index.pendingVectorBackfill} documents are waiting for a vector embedding.`, + fix: "This runs in the background and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. If it is stuck at a nonzero count with no embedding provider errors, set AGENTMEMORY_VECTOR_BACKFILL=all to run a full backfill.", + }); + } + const persistence = input.indexPersistence ?? null; const vectorLeg = persistence?.vector ?? null; if (persistence && vectorLeg) { @@ -193,6 +213,15 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } } + if (persistence?.vectorCountShortfall) { + const { expected, loaded } = persistence.vectorCountShortfall; + problems.push({ + level: "warn", + code: "index-vector-count-shortfall", + message: `Only ${loaded} of ${expected} vectors loaded from the last save; a bounded backfill re-embeds the rest.`, + fix: "No action needed — missing vectors are queued for backfill up to the per-boot cap.", + }); + } let graph: StatusReport["graph"] = null; if (input.graph) { @@ -377,6 +406,8 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off") ${row("Observations indexed", escapeHtml(idx.observationsIndexed))} ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))} ${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))} +${row("BM25 rebuild", idx.bm25Incomplete ? 'incomplete' : "complete")} +${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))} ${indexPersistenceRows(report)}
BM25 savesaved 1m ago, unsaved changes pendingBM25 savesaved 1m ago, unsaved changes pendingBM25 indexrebuilt from stored content at bootVector savesaved 1m ago, unsaved changes pendingVector storage256 buckets, 3 unsaved changes

Knowledge graph

diff --git a/src/index.ts b/src/index.ts index 23c11a2cf8..c043a31fab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,6 +41,7 @@ import { setEmbeddingProvider, setIndexPersistence, setHybridRanker, + setPendingVectorBackfillCount, } from "./functions/search.js"; import { registerContextFunction } from "./functions/context.js"; import { registerSummarizeFunction } from "./functions/summarize.js"; @@ -456,10 +457,14 @@ async function main() { } } + const vectorCountShortfall = + Boolean(loaded?.vector) && + loaded?.expectedCount !== undefined && + loaded.vector!.size < loaded.expectedCount; const vectorBackfillSince = !loaded || loaded.state === "unavailable" ? undefined - : loaded.state === "none" + : loaded.state === "none" || vectorCountShortfall ? null : loaded.savedAt; const keywordStart = Date.now(); @@ -468,9 +473,18 @@ async function main() { bootLog( `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`, ); + setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending); + if (keyword.fullBackfillPending > 0) { + bootLog( + `Vector backfill needs ${keyword.fullBackfillPending} embeddings but a full backfill was not started ` + + `(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`, + ); + } if (keyword.vectorJobs.length > 0) { + bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`); void backfillVectors(keyword.vectorJobs) .then((count) => { + setPendingVectorBackfillCount(keyword.fullBackfillPending); if (count > 0) bootLog(`Vector index backfilled: ${count} entries`); }) .catch((err) => { diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 4658e44371..97ea773189 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -3,7 +3,7 @@ import type { StateKV } from "./kv.js"; import { KV } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; -import { getIndexSaveIntervalMs, getVectorBucketCount } from "../config.js"; +import { getIndexSaveIntervalMs, getVectorBucketSize } from "../config.js"; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; @@ -31,8 +31,8 @@ type IndexShardManifest = { }; type VectorMeta = { - v: 2; - buckets: number; + v: 3; + bucketCount: number; savedAt: string; count: number; }; @@ -45,7 +45,7 @@ type PersistedVector = { type IndexPersistenceOptions = { saveIntervalMs?: number; - buckets?: number; + bucketSize?: number; now?: () => number; }; @@ -56,12 +56,18 @@ export interface IndexLegStatus { dirtySince: string | null; } +export interface VectorCountShortfall { + expected: number; + loaded: number; +} + export interface IndexPersistenceStatus { saveIntervalMs: number; saving: boolean; buckets: number; pendingChanges: number; vector: IndexLegStatus | null; + vectorCountShortfall: VectorCountShortfall | null; } export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable"; @@ -70,15 +76,7 @@ export interface VectorLoadResult { vector: VectorIndex | null; state: VectorLoadState; savedAt: string | null; -} - -export function vectorBucketOf(id: string, buckets: number): number { - let hash = 0x811c9dc5; - for (let i = 0; i < id.length; i++) { - hash ^= id.charCodeAt(i); - hash = Math.imul(hash, 0x01000193) >>> 0; - } - return hash % buckets; + expectedCount?: number; } export function vectorBucketScope(bucket: number): string { @@ -148,8 +146,13 @@ export class IndexPersistence { private metaWritten = false; private leg: IndexLegStatus = emptyLegStatus(); private readonly saveIntervalMs: number; - private buckets: number; + private readonly bucketSize: number; private readonly now: () => number; + private bucketOfId: Map = new Map(); + private bucketCounts: Map = new Map(); + private highestBucket = 0; + private hasOpenBucket = false; + private vectorCountShortfall: VectorCountShortfall | null = null; constructor( private kv: StateKV, @@ -162,11 +165,11 @@ export class IndexPersistence { typeof interval === "number" && Number.isFinite(interval) && interval > 0 ? interval : getIndexSaveIntervalMs(); - const buckets = options.buckets; - this.buckets = - typeof buckets === "number" && Number.isInteger(buckets) && buckets > 0 - ? buckets - : getVectorBucketCount(); + const bucketSize = options.bucketSize; + this.bucketSize = + typeof bucketSize === "number" && Number.isInteger(bucketSize) && bucketSize > 0 + ? bucketSize + : getVectorBucketSize(); this.lastSaveAt = this.now(); } @@ -202,9 +205,10 @@ export class IndexPersistence { return { saveIntervalMs: this.saveIntervalMs, saving: this.running !== null, - buckets: this.buckets, + buckets: this.bucketCountInUse(), pendingChanges: this.vector?.pendingChanges ?? 0, vector: this.vector ? { ...this.leg } : null, + vectorCountShortfall: this.vectorCountShortfall, }; } @@ -227,19 +231,55 @@ export class IndexPersistence { return { vector: null, state: "unavailable", savedAt: null }; } - if (meta && meta.v === 2 && Number.isInteger(meta.buckets) && meta.buckets > 0) { - const loaded = await this.loadBuckets(meta.buckets); + if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) { + const loaded = await this.loadBuckets(meta.bucketCount); if (!loaded) return { vector: null, state: "unavailable", savedAt: null }; this.metaWritten = true; - if (meta.buckets !== this.buckets) await this.moveBuckets(loaded, meta.buckets); await this.removeLegacyVectorSnapshotIfPresent(); await this.removeLegacyBm25Snapshot(); - return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null }; + const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined; + this.vectorCountShortfall = + expectedCount !== undefined && loaded.size < expectedCount + ? { expected: expectedCount, loaded: loaded.size } + : null; + if (this.vectorCountShortfall) { + logger.warn("index persistence: loaded fewer vectors than the last save recorded", { + ...this.vectorCountShortfall, + }); + } + return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount }; } return this.migrateLegacy(); } + private bucketCountInUse(): number { + return this.hasOpenBucket ? this.highestBucket + 1 : 0; + } + + private assignBucket(id: string): number { + const existing = this.bucketOfId.get(id); + if (existing !== undefined) return existing; + if (!this.hasOpenBucket) { + this.hasOpenBucket = true; + this.highestBucket = 0; + } else if ((this.bucketCounts.get(this.highestBucket) ?? 0) >= this.bucketSize) { + this.highestBucket++; + } + this.bucketOfId.set(id, this.highestBucket); + this.bucketCounts.set(this.highestBucket, (this.bucketCounts.get(this.highestBucket) ?? 0) + 1); + return this.highestBucket; + } + + private releaseBucket(id: string): void { + const bucket = this.bucketOfId.get(id); + if (bucket === undefined) return; + this.bucketOfId.delete(id); + const remaining = (this.bucketCounts.get(bucket) ?? 1) - 1; + if (remaining > 0) this.bucketCounts.set(bucket, remaining); + else this.bucketCounts.delete(bucket); + } + private clearTimer(): void { if (this.timer) { clearTimeout(this.timer); @@ -284,17 +324,21 @@ export class IndexPersistence { if (changes.size === 0 && this.metaWritten) return; const failed = new Map(); const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => { - const scope = vectorBucketScope(vectorBucketOf(id, this.buckets)); const entry = present ? vector.get(id) : undefined; try { if (entry) { - await this.kv.set(scope, id, { + const bucket = this.assignBucket(id); + await this.kv.set(vectorBucketScope(bucket), id, { id, s: entry.sessionId, e: float32ToBase64(entry.embedding), }); } else { - await this.kv.delete(scope, id); + const bucket = this.bucketOfId.get(id); + if (bucket !== undefined) { + await this.kv.delete(vectorBucketScope(bucket), id); + this.releaseBucket(id); + } } } catch (err) { failed.set(id, present); @@ -308,19 +352,22 @@ export class IndexPersistence { ); } await this.kv.set(KV.bm25Index, VECTOR_META_KEY, { - v: 2, - buckets: this.buckets, + v: 3, + bucketCount: this.bucketCountInUse(), savedAt: new Date(this.now()).toISOString(), count: vector.size, }); this.metaWritten = true; } - private async loadBuckets(buckets: number): Promise { + private async loadBuckets(bucketCount: number): Promise { const loaded = new VectorIndex(); - const bucketIds = Array.from({ length: buckets }, (_, bucket) => bucket); + const bucketOfId = new Map(); + const bucketCounts = new Map(); + const bucketIds = Array.from({ length: bucketCount }, (_, bucket) => bucket); const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => { const rows = await this.kv.list(vectorBucketScope(bucket)); + let count = 0; for (const row of rows) { if (!isPersistedVector(row)) continue; try { @@ -328,7 +375,10 @@ export class IndexPersistence { } catch { continue; } + bucketOfId.set(row.id, bucket); + count++; } + if (count > 0) bucketCounts.set(bucket, count); }); if (failures.length > 0) { logger.warn("index persistence: vector bucket read failed", { @@ -337,37 +387,13 @@ export class IndexPersistence { }); return null; } + this.bucketOfId = bucketOfId; + this.bucketCounts = bucketCounts; + this.highestBucket = bucketCount > 0 ? bucketCount - 1 : 0; + this.hasOpenBucket = bucketCount > 0; return loaded; } - private async moveBuckets(vector: VectorIndex, fromBuckets: number): Promise { - const moves = [...vector.entries()].filter( - ([id]) => vectorBucketOf(id, fromBuckets) !== vectorBucketOf(id, this.buckets), - ); - const failures = await inBatches(moves, WRITE_CONCURRENCY, async ([id, entry]) => { - await this.kv.set(vectorBucketScope(vectorBucketOf(id, this.buckets)), id, { - id, - s: entry.sessionId, - e: float32ToBase64(entry.embedding), - }); - await this.kv.delete(vectorBucketScope(vectorBucketOf(id, fromBuckets)), id); - }); - if (failures.length > 0) { - logger.warn("index persistence: moving vectors to the new bucket count failed; keeping the old count", { - failed: failures.length, - message: errorMessage(failures[0]), - }); - this.buckets = fromBuckets; - return; - } - await this.kv.set(KV.bm25Index, VECTOR_META_KEY, { - v: 2, - buckets: this.buckets, - savedAt: new Date(this.now()).toISOString(), - count: vector.size, - }); - } - private async migrateLegacy(): Promise { const manifest = await this.readIndexValue( KV.bm25Index, @@ -409,11 +435,11 @@ export class IndexPersistence { await this.removeLegacyBm25Snapshot(); await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], { vectors: legacy.size, - buckets: this.buckets, + buckets: this.bucketCountInUse(), }); logger.info("index persistence: migrated the vector index to bucketed storage", { vectors: legacy.size, - buckets: this.buckets, + buckets: this.bucketCountInUse(), }); const migrated = new VectorIndex(); for (const [id, entry] of legacy.entries()) migrated.loadPersisted(id, entry.sessionId, entry.embedding); diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 2fc2f3285a..337e34b5d5 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -12,7 +12,14 @@ import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js"; -import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js"; +import { + findUnindexedObservations, + getIndexPersistenceStatus, + getPendingVectorBackfillCount, + getSearchIndex, + getVectorIndex, + isBm25RebuildIncomplete, +} from "../functions/search.js"; import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; @@ -370,6 +377,8 @@ export function registerApiTriggers( observationsIndexed, missingObservations: unindexed ? unindexed.missing.length : null, sessions: unindexed ? unindexed.sessions : null, + bm25Incomplete: isBm25RebuildIncomplete(), + pendingVectorBackfill: getPendingVectorBackfillCount(), }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.ts index 4778106bec..f0080675ba 100644 --- a/test/index-keyword-rebuild.test.ts +++ b/test/index-keyword-rebuild.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect, afterEach } from "vitest"; import { backfillVectors, + getPendingVectorBackfillCount, getSearchIndex, + isBm25RebuildIncomplete, isMemoryIndexReady, rebuildKeywordIndex, setEmbeddingProvider, @@ -142,6 +144,29 @@ describe("rebuildKeywordIndex", () => { expect(elapsed).toBeLessThan(10_000); }); + it("keeps the vector index untouched and reports an incomplete rebuild when the sessions listing fails", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + const vector = new VectorIndex(); + vector.add("obs_0", "ses_0", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + const failingKv = { + ...kv, + list: async (scope: string): Promise => { + if (scope === "mem:sessions") throw new Error("engine unreachable"); + return kv.list(scope); + }, + }; + + const result = await rebuildKeywordIndex(failingKv as never, null); + + expect(result.vectorJobs).toEqual([]); + expect(vector.size).toBe(1); + expect(vector.has("obs_0")).toBe(true); + expect(isBm25RebuildIncomplete()).toBe(true); + }, 10_000); + it("queues embeddings only for documents newer than the last vector save", async () => { const kv = mockKV(); seed(kv, 2, 5, (i) => (i < 5 ? "2026-09-01T00:00:00.000Z" : "2026-09-20T00:00:00.000Z")); @@ -155,33 +180,61 @@ describe("rebuildKeywordIndex", () => { expect(result.vectorJobs.map((j) => j.id).sort()).toEqual(["obs_5", "obs_7", "obs_8", "obs_9"]); }); - it("queues every missing document when no vector state was persisted, and none when storage was unavailable", async () => { + it("withholds a whole-store backfill by default and reports it as pending, and queues none when storage was unavailable", async () => { const kv = mockKV(); seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); setVectorIndex(new VectorIndex()); setEmbeddingProvider(stubProvider()); const fresh = await rebuildKeywordIndex(kv as never, null); - expect(fresh.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]); + expect(fresh.vectorJobs).toEqual([]); + expect(fresh.fullBackfillPending).toBe(4); const unavailable = await rebuildKeywordIndex(kv as never, undefined); expect(unavailable.vectorJobs).toEqual([]); + expect(unavailable.fullBackfillPending).toBe(0); }); - it("backfills queued vectors in batches and schedules a save", async () => { + it("runs the whole-store backfill, capped, once AGENTMEMORY_VECTOR_BACKFILL=all is set", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + setVectorIndex(new VectorIndex()); + setEmbeddingProvider(stubProvider()); + process.env.AGENTMEMORY_VECTOR_BACKFILL = "all"; + try { + const opted = await rebuildKeywordIndex(kv as never, null); + expect(opted.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]); + expect(opted.fullBackfillPending).toBe(0); + + process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "2"; + const capped = await rebuildKeywordIndex(kv as never, null); + expect(capped.vectorJobs.length).toBe(2); + } finally { + delete process.env.AGENTMEMORY_VECTOR_BACKFILL; + delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX; + } + }); + + it("backfills queued vectors in batches and saves progress", async () => { const kv = mockKV(); seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); const vector = new VectorIndex(); setVectorIndex(vector); setEmbeddingProvider(stubProvider()); - let scheduled = 0; - setIndexPersistence({ scheduleSave: () => void scheduled++, save: async () => {} }); - - const { vectorJobs } = await rebuildKeywordIndex(kv as never, null); - const added = await backfillVectors(vectorJobs); - - expect(added).toBe(4); - expect(vector.size).toBe(4); - expect(scheduled).toBe(1); + let saved = 0; + setIndexPersistence({ scheduleSave: () => {}, save: async () => void saved++ }); + process.env.AGENTMEMORY_VECTOR_BACKFILL = "all"; + + try { + const { vectorJobs } = await rebuildKeywordIndex(kv as never, null); + const added = await backfillVectors(vectorJobs); + + expect(added).toBe(4); + expect(vector.size).toBe(4); + expect(saved).toBe(1); + expect(getPendingVectorBackfillCount()).toBe(0); + } finally { + delete process.env.AGENTMEMORY_VECTOR_BACKFILL; + } }); }); diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts index 09e0572981..bc03834d41 100644 --- a/test/index-persistence-throttle.test.ts +++ b/test/index-persistence-throttle.test.ts @@ -46,7 +46,7 @@ describe("IndexPersistence save throttling", () => { it("saves at most once per interval however often changes are scheduled", async () => { const vector = new VectorIndex(); - const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); for (let i = 0; i < 50; i++) { touch(vector, `obs_${i}`); @@ -70,7 +70,7 @@ describe("IndexPersistence save throttling", () => { it("an explicit save runs immediately and cancels the pending timer", async () => { const vector = new VectorIndex(); - const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); touch(vector, "obs_1"); persistence.scheduleSave(); @@ -101,7 +101,7 @@ describe("IndexPersistence save throttling", () => { }; const vector = new VectorIndex(); touch(vector, "obs_1"); - const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); const first = persistence.save(); touch(vector, "obs_2"); @@ -137,7 +137,7 @@ describe("IndexPersistence save throttling", () => { }; const vector = new VectorIndex(); touch(vector, "obs_1"); - const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, buckets: 16 }); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); const saving = persistence.save(); touch(vector, "obs_2"); @@ -155,7 +155,7 @@ describe("IndexPersistence save throttling", () => { it("stop prevents later scheduled saves", async () => { const vector = new VectorIndex(); touch(vector, "obs_1"); - const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, bucketSize: 16 }); persistence.stop(); persistence.scheduleSave(); await vi.advanceTimersByTimeAsync(10_000); @@ -176,7 +176,15 @@ function statusInputs(overrides: Partial = {}): StatusInputs { provider: "llm", embeddingProvider: "embeddings", flags: [], - index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 }, + index: { + bm25Documents: 1, + vectorDocuments: 1, + observationsIndexed: 1, + missingObservations: 0, + sessions: 1, + bm25Incomplete: false, + pendingVectorBackfill: 0, + }, graph: null, graphExtractionEnabled: false, ...overrides, @@ -191,7 +199,7 @@ const cleanLeg = { }; function persistenceStatus(vector: typeof cleanLeg | null, pendingChanges = 0) { - return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector }; + return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector, vectorCountShortfall: null }; } describe("status reports index persistence", () => { diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index c469be8134..231ecb98fc 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { IndexPersistence, vectorBucketOf, vectorBucketScope } from "../src/state/index-persistence.js"; +import { IndexPersistence, vectorBucketScope } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; import type { CompressedObservation } from "../src/types.js"; @@ -115,21 +115,21 @@ describe("IndexPersistence bucketed vector storage", () => { kv = mockKV(); }); - it("round-trips vectors through fixed bucket scopes", async () => { + it("fills buckets in insertion order and rolls to a new one at the cap", async () => { const vector = vectorWith([ ["obs_a", [0.1, 0.2, 0.3]], ["obs_b", [0.4, 0.5, 0.6]], ["obs_c", [0.7, 0.8, 0.9]], ]); - await new IndexPersistence(kv as never, vector, { buckets: 16 }).save(); + await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save(); - for (const [id] of vector.entries()) { - expect(kv.store.get(vectorBucketScope(vectorBucketOf(id, 16)))?.has(id)).toBe(true); - } - const meta = await kv.get<{ v: number; buckets: number; count: number }>(INDEX_SCOPE, META_KEY); - expect(meta).toMatchObject({ v: 2, buckets: 16, count: 3 }); + expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true); + expect(kv.store.get(vectorBucketScope(0))?.has("obs_b")).toBe(true); + expect(kv.store.get(vectorBucketScope(1))?.has("obs_c")).toBe(true); + const meta = await kv.get<{ v: number; bucketCount: number; count: number }>(INDEX_SCOPE, META_KEY); + expect(meta).toMatchObject({ v: 3, bucketCount: 2, count: 3 }); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }).load(); expect(loaded.state).toBe("buckets"); expectSameVectors(loaded.vector, vector); expect(loaded.vector!.pendingChanges).toBe(0); @@ -140,7 +140,7 @@ describe("IndexPersistence bucketed vector storage", () => { ["obs_a", [0.1, 0.2, 0.3]], ["obs_b", [0.4, 0.5, 0.6]], ]); - const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); await persistence.save(); kv.ops.length = 0; @@ -148,12 +148,12 @@ describe("IndexPersistence bucketed vector storage", () => { await persistence.save(); const writes = bucketSets(kv); - expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(vectorBucketOf("obs_new", 16)), key: "obs_new" }]); + expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(0), key: "obs_new" }]); }); it("does not write when nothing changed", async () => { const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); - const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); await persistence.save(); kv.ops.length = 0; @@ -167,31 +167,47 @@ describe("IndexPersistence bucketed vector storage", () => { ["obs_a", [0.1, 0.2, 0.3]], ["obs_b", [0.4, 0.5, 0.6]], ]); - const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); await persistence.save(); vector.remove("obs_a"); await persistence.save(); - expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(vectorBucketOf("obs_a", 16)), key: "obs_a" }); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(0), key: "obs_a" }); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded.vector!.has("obs_a")).toBe(false); expect(loaded.vector!.has("obs_b")).toBe(true); }); + it("a delete touches only the bucket that owns the id, in a multi-bucket store", async () => { + const vector = vectorWith( + Array.from({ length: 5 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + ); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 2 }); + await persistence.save(); + kv.ops.length = 0; + + vector.remove("obs_2"); + await persistence.save(); + + const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(1)])); + expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(1), key: "obs_2" }); + }); + it("clearing the index deletes every persisted vector", async () => { const vector = vectorWith([ ["obs_a", [0.1, 0.2, 0.3]], ["obs_b", [0.4, 0.5, 0.6]], ]); - const persistence = new IndexPersistence(kv as never, vector, { buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); await persistence.save(); vector.clear(); vector.add("obs_c", "ses_c", vec([0, 1, 0])); await persistence.save(); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect([...loaded.vector!.entries()].map(([id]) => id)).toEqual(["obs_c"]); }); @@ -205,7 +221,7 @@ describe("IndexPersistence bucketed vector storage", () => { return kv.set(scope, key, data); }, }; - const persistence = new IndexPersistence(flakyKv as never, vector, { buckets: 16 }); + const persistence = new IndexPersistence(flakyKv as never, vector, { bucketSize: 16 }); await expect(persistence.save()).resolves.toBeUndefined(); const failed = persistence.status(); @@ -217,27 +233,68 @@ describe("IndexPersistence bucketed vector storage", () => { await persistence.save(); expect(persistence.status().vector?.lastError).toBeNull(); expect(persistence.status().pendingChanges).toBe(0); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expectSameVectors(loaded.vector, vector); }); - it("moves vectors when the bucket count changes", async () => { + it("a save after many vectors added over several waves touches only the currently open bucket", async () => { + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 100 }); + + for (let i = 0; i < 950; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0])); + await persistence.save(); + expect(kv.store.get(vectorBucketScope(9))?.size).toBe(50); + for (let bucket = 0; bucket <= 8; bucket++) expect(kv.store.get(vectorBucketScope(bucket))?.size).toBe(100); + kv.ops.length = 0; + + for (let i = 950; i < 1000; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0])); + await persistence.save(); + let touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(9)])); + expect(kv.store.get(vectorBucketScope(9))?.size).toBe(100); + kv.ops.length = 0; + + vector.add("obs_1000", "ses_1", vec([1000, 0, 0])); + await persistence.save(); + touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(10)])); + }); + + it("a smaller bucket size configured later opens a new bucket instead of moving existing data", async () => { const vector = vectorWith( - Array.from({ length: 40 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), ); - await new IndexPersistence(kv as never, vector, { buckets: 4 }).save(); + await new IndexPersistence(kv as never, vector, { bucketSize: 4 }).save(); + for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 8 }).load(); - expectSameVectors(loaded.vector, vector); - for (const [id] of vector.entries()) { - const oldScope = vectorBucketScope(vectorBucketOf(id, 4)); - const newScope = vectorBucketScope(vectorBucketOf(id, 8)); - expect(kv.store.get(newScope)?.has(id)).toBe(true); - if (oldScope !== newScope) expect(kv.store.get(oldScope)?.has(id) ?? false).toBe(false); - } + const reloaded = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, reloaded, { bucketSize: 2 }); + const loaded = await persistence.load(); + reloaded.restoreFrom(loaded.vector!); + kv.ops.length = 0; + + reloaded.add("obs_new", "ses_new", vec([9, 9, 9])); + await persistence.save(); - const again = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 8 }).load(); - expect(again.vector!.size).toBe(40); + const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(1)])); + for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true); + }); + + it("detects a partial bucket set against the saved count and reports it in status", async () => { + const vector = vectorWith( + Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + ); + await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save(); + kv.store.get(vectorBucketScope(1))?.clear(); + + const persistence = new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }); + const loaded = await persistence.load(); + + expect(loaded.state).toBe("buckets"); + expect(loaded.vector!.size).toBe(2); + expect(loaded.expectedCount).toBe(4); + expect(persistence.status().vectorCountShortfall).toEqual({ expected: 4, loaded: 2 }); }); it("reports storage as unavailable when the metadata read fails", async () => { @@ -247,13 +304,13 @@ describe("IndexPersistence bucketed vector storage", () => { throw new Error("engine down"); }, }; - const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null }); }); it("treats a missing store as empty, including adapters that return undefined", async () => { const undefinedKv = { ...kv, get: async () => undefined }; - const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); }); }); @@ -274,7 +331,7 @@ describe("IndexPersistence migration from the single-string format", () => { await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); await writeLegacyBm25Snapshot(kv); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded.state).toBe("migrated"); expect(loaded.savedAt).toBe(new Date(parseInt("mfabcd12", 36)).toISOString()); @@ -286,7 +343,7 @@ describe("IndexPersistence migration from the single-string format", () => { if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0); } - const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(reloaded.state).toBe("buckets"); expectSameVectors(reloaded.vector, legacy); }); @@ -295,7 +352,7 @@ describe("IndexPersistence migration from the single-string format", () => { const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true }); - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded.state).toBe("migrated"); expectSameVectors(loaded.vector, legacy); @@ -309,7 +366,7 @@ describe("IndexPersistence migration from the single-string format", () => { await kv.delete(manifest!.shards[0].scope, manifest!.shards[0].key); kv.ops.length = 0; - const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(loaded.state).toBe("unavailable"); expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); @@ -332,7 +389,7 @@ describe("IndexPersistence migration from the single-string format", () => { }; const live = new VectorIndex(); - const persistence = new IndexPersistence(flakyKv as never, live, { buckets: 16 }); + const persistence = new IndexPersistence(flakyKv as never, live, { bucketSize: 16 }); const loaded = await persistence.load(); expect(loaded.state).toBe("unavailable"); expectSameVectors(loaded.vector, legacy); @@ -343,7 +400,7 @@ describe("IndexPersistence migration from the single-string format", () => { fail = false; await persistence.save(); - const next = await new IndexPersistence(kv as never, new VectorIndex(), { buckets: 16 }).load(); + const next = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); expect(next.state).toBe("buckets"); expectSameVectors(next.vector, legacy); expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull(); @@ -354,7 +411,7 @@ describe("IndexPersistence migration from the single-string format", () => { await writeLegacyVectorSnapshot(kv, legacy); await writeLegacyBm25Snapshot(kv); - const loaded = await new IndexPersistence(kv as never, null, { buckets: 16 }).load(); + const loaded = await new IndexPersistence(kv as never, null, { bucketSize: 16 }).load(); expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull(); @@ -384,7 +441,7 @@ describe("IndexPersistence scheduling", () => { }), }; const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); - const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, buckets: 16 }); + const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 }); let unhandled = false; const onUnhandled = () => { unhandled = true; @@ -402,7 +459,7 @@ describe("IndexPersistence scheduling", () => { it("stop clears the pending timer", async () => { const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); - const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, buckets: 16 }); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 }); persistence.scheduleSave(); persistence.stop(); await vi.advanceTimersByTimeAsync(10_000); diff --git a/test/search.test.ts b/test/search.test.ts index 84b8134e3a..99b7f4d6d3 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -209,4 +209,21 @@ describe("mem::search", () => { setVectorIndex(null); setEmbeddingProvider(null); }); + + it("a cold-start search rebuilds BM25 without touching the persisted vector index", async () => { + const vector = new VectorIndex(); + vector.add("obs_persisted", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + getSearchIndex().clear(); + + const result = (await sdk.trigger("mem::search", { + query: "auth middleware", + })) as { results: Array<{ observation: CompressedObservation }> }; + + expect(result.results[0]?.observation.id).toBe("obs_a"); + expect(getVectorIndex()?.has("obs_persisted")).toBe(true); + expect(getVectorIndex()?.size).toBe(1); + + setVectorIndex(null); + }); }); diff --git a/test/status.test.ts b/test/status.test.ts index 6ecfb50ca5..2160a65f92 100644 --- a/test/status.test.ts +++ b/test/status.test.ts @@ -27,6 +27,8 @@ function inputs(overrides: Partial = {}): StatusInputs { observationsIndexed: 110, missingObservations: 0, sessions: 95, + bm25Incomplete: false, + pendingVectorBackfill: 0, }, graph: { totalNodes: 55, @@ -93,6 +95,28 @@ describe("evaluateStatus", () => { expect(report.status).toBe("info"); }); + it("reports an incomplete BM25 rebuild as an error", () => { + const report = evaluateStatus(inputs({ index: { ...inputs().index, bm25Incomplete: true } })); + expect(report.status).toBe("error"); + expect(codes(report)).toContain("bm25-rebuild-incomplete"); + }); + + it("notes a pending vector backfill without raising the overall status", () => { + const report = evaluateStatus(inputs({ index: { ...inputs().index, pendingVectorBackfill: 42 } })); + expect(report.status).toBe("info"); + const problem = report.problems.find((p) => p.code === "index-vector-backfill-pending"); + expect(problem?.message).toBe("42 documents are waiting for a vector embedding."); + }); + + it("reports a vector count shortfall from the last save as a warning", () => { + const report = evaluateStatus( + inputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, buckets: 3, pendingChanges: 0, vector: null, vectorCountShortfall: { expected: 100, loaded: 40 } } }), + ); + expect(report.status).toBe("warn"); + const problem = report.problems.find((p) => p.code === "index-vector-count-shortfall"); + expect(problem?.message).toContain("40 of 100 vectors"); + }); + it("checks snapshot presence when extraction is on, and snapshot age whenever a snapshot exists", () => { const noSnapshot = { totalNodes: 0, totalEdges: 0, fromSnapshot: false }; expect(codes(evaluateStatus(inputs({ graph: noSnapshot })))).toEqual(["graph-no-snapshot"]); From 5c9ffc546942de35a329ae2055e41526669b4193 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 13:29:49 +0530 Subject: [PATCH 7/7] fix(index): remove legacy BM25 shards when missing keys read back undefined The engine SDK returns undefined for a missing key. The legacy cleanup treated undefined as a failed read and returned early, so after migration the old BM25 shards and manifest stayed in the store for good. Both legacy cleanups now read through readIndexValue, which only reports failure when the read throws. --- src/state/index-persistence.ts | 22 ++++++++++------------ test/index-persistence.test.ts | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 97ea773189..d046d0dd0f 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -455,21 +455,19 @@ export class IndexPersistence { } private async removeLegacyVectorSnapshotIfPresent(): Promise { - const manifest = await this.kv - .get(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY) - .catch(() => undefined); - const legacy = await this.kv.get(KV.bm25Index, LEGACY_VECTOR_KEY).catch(() => undefined); - if (manifest === undefined || legacy === undefined) return; - if (manifest == null && legacy == null) return; - await this.removeLegacyVectorSnapshot(manifest); + const manifest = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "vector", "manifest"); + const legacy = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy"); + if (!manifest.ok || !legacy.ok) return; + if (manifest.value == null && legacy.value == null) return; + await this.removeLegacyVectorSnapshot(manifest.value ?? null); } private async removeLegacyBm25Snapshot(): Promise { - const manifest = await this.kv - .get(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY) - .catch(() => undefined); - const legacy = await this.kv.get(KV.bm25Index, LEGACY_BM25_KEY).catch(() => undefined); - if (manifest === undefined || legacy === undefined) return; + const manifestRead = await this.readIndexValue(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "bm25", "manifest"); + const legacyRead = await this.readIndexValue(KV.bm25Index, LEGACY_BM25_KEY, "bm25", "legacy"); + if (!manifestRead.ok || !legacyRead.ok) return; + const manifest = manifestRead.value ?? null; + const legacy = legacyRead.value ?? null; if (manifest == null && legacy == null) return; if (manifest && Array.isArray(manifest.shards)) { await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_bm25_cleanup"); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 231ecb98fc..568d180006 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -348,6 +348,26 @@ describe("IndexPersistence migration from the single-string format", () => { expectSameVectors(reloaded.vector, legacy); }); + it("removes the old BM25 shards when missing keys read back as undefined, as the engine SDK returns them", async () => { + const baseGet = kv.get; + kv.get = (async (scope: string, key: string) => { + const value = await baseGet(scope, key); + return value === null ? undefined : value; + }) as typeof kv.get; + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_bbbbbbbbbbbb" }); + await writeLegacyBm25Snapshot(kv); + + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + + expect(loaded.state).toBe("migrated"); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeUndefined(); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeUndefined(); + for (const [scope, entries] of kv.store) { + if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0); + } + }); + it("migrates a monolithic vector snapshot", async () => { const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true });