From 3ca6c26ec1af2c713186f9cffbd27790e25b040e Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Fri, 25 Sep 2026 19:13:11 +0530 Subject: [PATCH 1/3] perf(index): throttle index saves and report each leg's save state Every observation, compression and memory save scheduled a full index save 5 s later, and each save re-serializes the whole BM25 and vector index and rewrites every shard. On a busy store that is a full index rewrite almost continuously. Saves now run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS (default 10 minutes), measured from the start of the previous save. Explicit saves (deletes, shutdown) still run immediately. Only one save runs at a time; saves requested while one is running collapse into a single follow-up save. The BM25 and vector saves now fail independently. Before, a BM25 failure skipped the vector save in the same run and both were reported as one warning. Each leg records when it last saved, its last error, since when it has unsaved changes, and its serialized size. /agentmemory/status reports a failing leg as an error, unsaved changes older than twice the interval as a warning, and a vector index above 80% of the Node.js string limit (536,870,888 characters) as a warning, since past that limit the index can no longer be saved or loaded. The status page shows each leg's save state and the vector index size. --- .env.example | 1 + plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/config.ts | 10 + src/functions/search.ts | 16 +- src/functions/status.ts | 67 ++++ src/state/index-persistence.ts | 172 ++++++++-- src/triggers/api.ts | 3 +- test/index-persistence-throttle.test.ts | 302 ++++++++++++++++++ test/index-persistence.test.ts | 6 +- 9 files changed, 539 insertions(+), 41 deletions(-) create mode 100644 test/index-persistence-throttle.test.ts diff --git a/.env.example b/.env.example index 9219a338a1..4b9deb0b60 100644 --- a/.env.example +++ b/.env.example @@ -129,6 +129,7 @@ # AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). +# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index d152a7c62b..5fe071567b 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 43 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -27,6 +27,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` - `AGENTMEMORY_IMAGE_STORE_MAX_BYTES` +- `AGENTMEMORY_INDEX_SAVE_INTERVAL_MS` - `AGENTMEMORY_INJECT_CONTEXT` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` diff --git a/src/config.ts b/src/config.ts index a6c62ce55e..ac82bd445f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -467,6 +467,16 @@ export function getConsolidationCooldownMs(): number { return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS; } +export const INDEX_SAVE_INTERVAL_DEFAULT_MS = 600_000; + +export function getIndexSaveIntervalMs(): number { + const raw = safeParseInt( + getMergedEnv()["AGENTMEMORY_INDEX_SAVE_INTERVAL_MS"], + INDEX_SAVE_INTERVAL_DEFAULT_MS, + ); + return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/search.ts b/src/functions/search.ts index 2b8858bdc5..8abce07f45 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -1,4 +1,5 @@ import type { IIIClient } from 'iii-sdk' +import type { IndexPersistenceStatus } from "../state/index-persistence.js"; import type { CompactSearchResult, CompressedObservation, Memory, SearchResult, Session } from '../types.js' import { KV } from '../state/schema.js' import { StateKV } from '../state/kv.js' @@ -75,17 +76,22 @@ export function vectorIndexRemove(id: string): void { // Wired by src/index.ts after IndexPersistence is constructed; no-op // until then so unit tests that exercise the delete paths in // isolation don't need to wire persistence. -let indexPersistence: { +type IndexPersistenceHook = { scheduleSave: () => void; save: () => Promise; -} | null = null; + status?: () => IndexPersistenceStatus; +}; -export function setIndexPersistence( - p: { scheduleSave: () => void; save: () => Promise } | null, -): void { +let indexPersistence: IndexPersistenceHook | null = null; + +export function setIndexPersistence(p: IndexPersistenceHook | null): void { indexPersistence = p; } +export function getIndexPersistenceStatus(): IndexPersistenceStatus | null { + return indexPersistence?.status?.() ?? null; +} + export function scheduleIndexSave(): void { indexPersistence?.scheduleSave(); } diff --git a/src/functions/status.ts b/src/functions/status.ts index 552fe6c07a..0b3ab5310a 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -1,3 +1,5 @@ +import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js"; + export type StatusLevel = "ok" | "info" | "warn" | "error"; export interface StatusProblem { @@ -58,6 +60,7 @@ export interface StatusInputs { }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; + indexPersistence?: IndexPersistenceStatus | null; } export interface StatusReport { @@ -72,6 +75,7 @@ export interface StatusReport { health: StatusInputs["health"]; provider: { llm: string; embeddings: string; circuitBreaker: StatusInputs["circuitBreaker"] }; index: StatusInputs["index"]; + indexPersistence: IndexPersistenceStatus | null; graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null; functions: Array; flags: StatusFlag[]; @@ -81,6 +85,7 @@ export interface StatusReport { const FAILURE_RATE_THRESHOLD = 0.2; const FAILURE_MIN_CALLS = 5; const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60; +const VECTOR_STRING_WARN_RATIO = 0.8; const LEVEL_RANK: Record = { ok: 0, info: 1, warn: 2, error: 3 }; const FUNCTION_FIXES: Record = { @@ -168,6 +173,43 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } + const persistence = input.indexPersistence ?? null; + if (persistence) { + const legs: Array<[string, IndexLegStatus | null]> = [ + ["BM25", persistence.bm25], + ["vector", persistence.vector], + ]; + for (const [label, leg] of legs) { + if (!leg) continue; + if (leg.lastError) { + problems.push({ + level: "error", + code: "index-save-failing", + message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`, + fix: "Check the server log for the failing state write. The next save retries automatically.", + }); + } + const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined); + if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { + problems.push({ + level: "warn", + code: "index-save-stale", + message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, + fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", + }); + } + } + const vectorChars = persistence.vector?.serializedChars ?? null; + if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) { + problems.push({ + level: "warn", + code: "vector-index-near-string-limit", + message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`, + fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.", + }); + } + } + let graph: StatusReport["graph"] = null; if (input.graph) { const ageSeconds = secondsBetween(input.now, input.graph.updatedAt); @@ -218,6 +260,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport { circuitBreaker: input.circuitBreaker, }, index: input.index, + indexPersistence: persistence, graph, functions, flags: input.flags, @@ -248,6 +291,29 @@ function row(label: string, value: string): string { return `${escapeHtml(label)}${value}`; } +function legSummary(report: StatusReport, leg: IndexLegStatus): string { + if (leg.lastError) return `failing: ${leg.lastError}`; + const saved = leg.lastSavedAt ? `saved ${formatDuration(secondsBetween(new Date(report.checkedAt), leg.lastSavedAt))} ago` : "not saved since start"; + return leg.dirtySince ? `${saved}, unsaved changes pending` : saved; +} + +function indexPersistenceRows(report: StatusReport): string { + const persistence = report.indexPersistence; + if (!persistence) return ""; + let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25))); + if (persistence.vector) { + rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector))); + const chars = persistence.vector.serializedChars; + if (chars !== null) { + rows += row( + "Vector index size", + escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`), + ); + } + } + return rows; +} + export function renderStatusHtml(report: StatusReport, styleNonce: string): string { const problems = report.problems.length ? report.problems @@ -330,6 +396,7 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off") ${row("Observations indexed", escapeHtml(idx.observationsIndexed))} ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))} ${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))} +${indexPersistenceRows(report)}

Knowledge graph

${graph diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 19b05cb598..a77d66ed30 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -4,8 +4,9 @@ import type { StateKV } from "./kv.js"; import { KV, generateId } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; +import { getIndexSaveIntervalMs } from "../config.js"; -const DEBOUNCE_MS = 5000; +export const V8_MAX_STRING_CHARS = 536_870_888; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; const BM25_KEY = "data"; @@ -41,8 +42,31 @@ type IndexShardManifest = { type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; + saveIntervalMs?: number; + now?: () => number; }; +export type IndexLeg = "bm25" | "vector"; + +export interface IndexLegStatus { + lastSavedAt: string | null; + lastError: string | null; + lastErrorAt: string | null; + dirtySince: string | null; + serializedChars: number | null; +} + +export interface IndexPersistenceStatus { + saveIntervalMs: number; + saving: boolean; + bm25: IndexLegStatus; + vector: IndexLegStatus | null; +} + +function emptyLegStatus(): IndexLegStatus { + return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null }; +} + function shardChars(options: IndexPersistenceOptions): number { const configured = options.shardChars; if (typeof configured !== "number" || !Number.isFinite(configured)) { @@ -81,39 +105,72 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private running: Promise | null = null; + private queued: Promise | null = null; + private stopped = false; + private lastSaveAt: number; + private dirtyEpoch = 0; + private markedDuringRunAt: number | null = null; + private legs: Record = { + bm25: emptyLegStatus(), + vector: emptyLegStatus(), + }; + private readonly saveIntervalMs: number; + private readonly now: () => number; constructor( private kv: StateKV, private bm25: SearchIndex, private vector: VectorIndex | null, private options: IndexPersistenceOptions = {}, - ) {} + ) { + this.now = options.now ?? Date.now; + const interval = options.saveIntervalMs; + this.saveIntervalMs = + typeof interval === "number" && Number.isFinite(interval) && interval > 0 + ? interval + : getIndexSaveIntervalMs(); + this.lastSaveAt = this.now(); + } scheduleSave(): void { - if (this.timer) clearTimeout(this.timer); - // setTimeout discards the returned promise, so any rejection inside - // save() would surface as unhandledRejection and crash the process - // under sustained iii-engine write timeouts (issue #204). Funnel - // rejections through logFailure() instead. + if (this.stopped) return; + const now = this.now(); + this.dirtyEpoch++; + if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now; + for (const leg of this.activeLegs()) { + if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString(); + } + if (this.timer) return; + const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now); this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); - }, DEBOUNCE_MS); + this.timer = null; + this.save().catch((err) => this.logFailure("bm25", err)); + }, delay); } - async save(): Promise { - if (this.timer) { - clearTimeout(this.timer); - this.timer = null; - } - try { - await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { - await this.saveVectorIndex(this.vector.serialize()); - } - } catch (err) { - this.logFailure(err); + save(): Promise { + this.clearTimer(); + if (this.queued) return this.queued; + if (this.running) { + const queued = this.running.then(() => { + this.queued = null; + return this.startRun(); + }); + this.queued = queued; + return queued; } + return this.startRun(); + } + + status(): IndexPersistenceStatus { + return { + saveIntervalMs: this.saveIntervalMs, + saving: this.running !== null, + bm25: { ...this.legs.bm25 }, + vector: this.vector ? { ...this.legs.vector } : null, + }; } async load(): Promise<{ @@ -137,27 +194,80 @@ export class IndexPersistence { } stop(): void { + this.stopped = true; + this.clearTimer(); + } + + private clearTimer(): void { if (this.timer) { clearTimeout(this.timer); this.timer = null; } } - private logFailure(err: unknown): void { - const now = Date.now(); - // Throttle: persistence failures under load arrive in bursts - // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + private activeLegs(): IndexLeg[] { + return this.vector ? ["bm25", "vector"] : ["bm25"]; + } + + private startRun(): Promise { + const run = this.runSave().finally(() => { + if (this.running === run) this.running = null; + }); + this.running = run; + return run; + } + + private async runSave(): Promise { + const epoch = this.dirtyEpoch; + this.markedDuringRunAt = null; + this.lastSaveAt = this.now(); + await this.saveLeg("bm25", epoch, async () => { + const serialized = this.bm25.serialize(); + this.legs.bm25.serializedChars = serialized.length; + await this.saveBm25Index(serialized); + }); + const vector = this.vector; + if (vector) { + await this.saveLeg("vector", epoch, async () => { + const serialized = vector.serialize(); + this.legs.vector.serializedChars = serialized.length; + await this.saveVectorIndex(serialized); + }); + } + } + + private async saveLeg(leg: IndexLeg, epoch: number, run: () => Promise): Promise { + const status = this.legs[leg]; + try { + await run(); + status.lastSavedAt = new Date(this.now()).toISOString(); + status.lastError = null; + status.lastErrorAt = null; + if (this.dirtyEpoch === epoch) { + status.dirtySince = null; + } else if (this.markedDuringRunAt !== null) { + status.dirtySince = new Date(this.markedDuringRunAt).toISOString(); + } + } catch (err) { + status.lastError = errorMessage(err); + status.lastErrorAt = new Date(this.now()).toISOString(); + if (status.dirtySince === null) status.dirtySince = status.lastErrorAt; + this.logFailure(leg, err); + } + } + + private logFailure(leg: IndexLeg, err: unknown): void { + const now = this.now(); + if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(leg, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, { code, message, hint: code === "TIMEOUT" - ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next debounce flush" + ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save" : undefined, }); } diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d8a447f2e0..2fc2f3285a 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -12,7 +12,7 @@ import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js"; -import { findUnindexedObservations, getSearchIndex, getVectorIndex } from "../functions/search.js"; +import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js"; import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; @@ -373,6 +373,7 @@ export function registerApiTriggers( }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), + indexPersistence: getIndexPersistenceStatus(), }); const accept = req.headers?.["accept"] ?? req.headers?.["Accept"]; const format = req.query_params?.["format"]; diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts new file mode 100644 index 0000000000..8a013fa662 --- /dev/null +++ b/test/index-persistence-throttle.test.ts @@ -0,0 +1,302 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js"; +import { SearchIndex } from "../src/state/search-index.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js"; +import type { CompressedObservation } from "../src/types.js"; + +const BM25_SCOPE = "mem:index:bm25"; + +function mockKV() { + const store = new Map>(); + const sets: Array<{ scope: string; key: string }> = []; + return { + sets, + store, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + sets.push({ scope, key }); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function obs(id: string, title: string): CompressedObservation { + return { + id, + sessionId: "ses_1", + timestamp: new Date().toISOString(), + type: "file_edit", + title, + facts: [], + narrative: `${title} narrative`, + concepts: [], + files: [], + importance: 5, + }; +} + +function manifestSaves(kv: ReturnType, key: string): number { + return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length; +} + +describe("IndexPersistence save throttling", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("saves at most once per interval however often changes are scheduled", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(59_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + + await vi.advanceTimersByTimeAsync(1_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(2); + }); + + it("an explicit save runs immediately and cancels the pending timer", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + await vi.advanceTimersByTimeAsync(120_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + }); + + it("never runs two saves at once and coalesces requests made during a save into one", async () => { + let release: () => void = () => undefined; + let inFlight = 0; + let maxInFlight = 0; + let gate: Promise | null = new Promise((resolve) => { + release = resolve; + }); + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + if (gate) await gate; + const result = await kv.set(scope, key, data); + inFlight--; + return result; + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const first = persistence.save(); + const second = persistence.save(); + const third = persistence.save(); + expect(second).toBe(third); + expect(persistence.status().saving).toBe(true); + + gate = null; + release(); + await Promise.all([first, second, third]); + + expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(maxInFlight).toBe(1); + expect(persistence.status().saving).toBe(false); + }); + + it("a failing BM25 leg does not stop the vector leg from saving", async () => { + const failingKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed"); + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + + const status = persistence.status(); + expect(status.bm25.lastError).toBe("bm25 manifest write failed"); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.lastError).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.serializedChars).toBeGreaterThan(0); + + const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + expect(loaded.bm25).toBeNull(); + expect(loaded.vector?.size).toBe(1); + }); + + it("keeps a leg dirty when a change arrives while it is being saved", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let held = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (!held) { + held = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + expect(persistence.status().bm25.lastSavedAt).not.toBeNull(); + expect(persistence.status().bm25.dirtySince).not.toBeNull(); + + await persistence.save(); + expect(persistence.status().bm25.dirtySince).toBeNull(); + }); + + it("stop prevents later scheduled saves", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 }); + persistence.stop(); + persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(10_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + }); +}); + +function statusInputs(overrides: Partial = {}): StatusInputs { + return { + now: new Date("2026-09-25T12:00:00.000Z"), + version: "0.9.29", + engineVersion: "0.22.1", + uptimeSeconds: 100, + ports: { rest: 3111, streams: 3112, viewer: 3113 }, + health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" }, + circuitBreaker: null, + functionMetrics: [], + provider: "llm", + embeddingProvider: "embeddings", + flags: [], + index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 }, + graph: null, + graphExtractionEnabled: false, + ...overrides, + }; +} + +const cleanLeg = { + lastSavedAt: "2026-09-25T11:59:00.000Z", + lastError: null, + lastErrorAt: null, + dirtySince: null, + serializedChars: 1000, +}; + +describe("status reports index persistence", () => { + it("is quiet when both legs saved cleanly", () => { + const report = evaluateStatus( + statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }), + ); + expect(report.problems).toEqual([]); + expect(report.indexPersistence?.vector?.serializedChars).toBe(1000); + }); + + it("reports a failing leg as an error", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" }, + }, + }), + ); + expect(report.status).toBe("error"); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]); + expect(report.problems[0].message).toContain("vector"); + }); + + it("warns when the vector index passes 80% of the Node.js string limit", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) }, + }, + }), + ); + expect(report.status).toBe("warn"); + expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]); + expect(report.problems[0].message).toContain("85%"); + }); + + it("warns when unsaved changes are older than twice the save interval", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" }, + vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" }, + }, + }), + ); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]); + expect(report.problems[0].message).toContain("BM25"); + }); + + it("shows save state and vector size on the status page", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" }, + vector: { ...cleanLeg, serializedChars: 53_687_089 }, + }, + }), + ); + const html = renderStatusHtml(report, "n"); + expect(html).toContain(""); + expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)"); + }); +}); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index ba47c3ef72..802084bd2d 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -670,7 +670,7 @@ describe("IndexPersistence", () => { it("scheduleSave debounces multiple calls", async () => { const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 }); persistence.scheduleSave(); persistence.scheduleSave(); @@ -721,7 +721,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); let unhandled = false; const onUnhandled = () => { @@ -751,7 +751,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); await expect(persistence.save()).resolves.toBeUndefined(); }); From 51f88a5fc89dec949e9988171d9276a3fb000f65 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:33:55 +0530 Subject: [PATCH 2/3] chore: credit co-authors The per-leg save failure handling and the single save queue build on the fix in #1256. The problem was measured and reported in discussion #1358. Co-authored-by: Omar Gerardo Co-authored-by: Vladimir <7998636+MarvinFS@users.noreply.github.com> From 14cb1a69288309db632659b30631ff30d0df98c0 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:42:39 +0530 Subject: [PATCH 3/3] fix(index-persistence): snapshot dirtyEpoch per leg before serializing runSave() captured one dirtyEpoch before the BM25 leg's write and reused it for the vector leg's dirty check. When scheduleSave() ran while the BM25 write was in flight, the vector leg would serialize the current (already up to date) state but still compare against the stale epoch, leaving vector.dirtySince set even though the just-written snapshot already reflected the change. status() would report the vector leg as dirty until an unrelated follow-up save cleared it. saveLeg() now snapshots dirtyEpoch itself, immediately before invoking the leg's own serialize+write callback, so each leg judges its dirty state against the epoch as of its own write rather than one captured before an earlier leg ran. Added a regression test that gates the BM25 manifest write, triggers a scheduleSave() while it is held, and asserts the vector leg's dirtySince clears once its own write completes. --- src/state/index-persistence.ts | 8 +++--- test/index-persistence-throttle.test.ts | 33 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index a77d66ed30..28a24d5a1c 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -218,17 +218,16 @@ export class IndexPersistence { } private async runSave(): Promise { - const epoch = this.dirtyEpoch; this.markedDuringRunAt = null; this.lastSaveAt = this.now(); - await this.saveLeg("bm25", epoch, async () => { + await this.saveLeg("bm25", async () => { const serialized = this.bm25.serialize(); this.legs.bm25.serializedChars = serialized.length; await this.saveBm25Index(serialized); }); const vector = this.vector; if (vector) { - await this.saveLeg("vector", epoch, async () => { + await this.saveLeg("vector", async () => { const serialized = vector.serialize(); this.legs.vector.serializedChars = serialized.length; await this.saveVectorIndex(serialized); @@ -236,8 +235,9 @@ export class IndexPersistence { } } - private async saveLeg(leg: IndexLeg, epoch: number, run: () => Promise): Promise { + private async saveLeg(leg: IndexLeg, run: () => Promise): Promise { const status = this.legs[leg]; + const epoch = this.dirtyEpoch; try { await run(); status.lastSavedAt = new Date(this.now()).toISOString(); diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts index 8a013fa662..54eff133b8 100644 --- a/test/index-persistence-throttle.test.ts +++ b/test/index-persistence-throttle.test.ts @@ -189,6 +189,39 @@ describe("IndexPersistence save throttling", () => { expect(persistence.status().bm25.dirtySince).toBeNull(); }); + it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let heldBm25Manifest = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) { + heldBm25Manifest = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + const status = persistence.status(); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + }); + it("stop prevents later scheduled saves", async () => { const bm25 = new SearchIndex(); bm25.add(obs("obs_1", "alpha"));
BM25 savesaved 1m ago, unsaved changes pending