diff --git a/.env.example b/.env.example index 9219a338a1..4b9deb0b60 100644 --- a/.env.example +++ b/.env.example @@ -129,6 +129,7 @@ # AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). +# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index d152a7c62b..5fe071567b 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 43 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -27,6 +27,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` - `AGENTMEMORY_IMAGE_STORE_MAX_BYTES` +- `AGENTMEMORY_INDEX_SAVE_INTERVAL_MS` - `AGENTMEMORY_INJECT_CONTEXT` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` diff --git a/src/config.ts b/src/config.ts index a6c62ce55e..ac82bd445f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -467,6 +467,16 @@ export function getConsolidationCooldownMs(): number { return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS; } +export const INDEX_SAVE_INTERVAL_DEFAULT_MS = 600_000; + +export function getIndexSaveIntervalMs(): number { + const raw = safeParseInt( + getMergedEnv()["AGENTMEMORY_INDEX_SAVE_INTERVAL_MS"], + INDEX_SAVE_INTERVAL_DEFAULT_MS, + ); + return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/search.ts b/src/functions/search.ts index 2b8858bdc5..8abce07f45 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -1,4 +1,5 @@ import type { IIIClient } from 'iii-sdk' +import type { IndexPersistenceStatus } from "../state/index-persistence.js"; import type { CompactSearchResult, CompressedObservation, Memory, SearchResult, Session } from '../types.js' import { KV } from '../state/schema.js' import { StateKV } from '../state/kv.js' @@ -75,17 +76,22 @@ export function vectorIndexRemove(id: string): void { // Wired by src/index.ts after IndexPersistence is constructed; no-op // until then so unit tests that exercise the delete paths in // isolation don't need to wire persistence. -let indexPersistence: { +type IndexPersistenceHook = { scheduleSave: () => void; save: () => Promise; -} | null = null; + status?: () => IndexPersistenceStatus; +}; -export function setIndexPersistence( - p: { scheduleSave: () => void; save: () => Promise } | null, -): void { +let indexPersistence: IndexPersistenceHook | null = null; + +export function setIndexPersistence(p: IndexPersistenceHook | null): void { indexPersistence = p; } +export function getIndexPersistenceStatus(): IndexPersistenceStatus | null { + return indexPersistence?.status?.() ?? null; +} + export function scheduleIndexSave(): void { indexPersistence?.scheduleSave(); } diff --git a/src/functions/status.ts b/src/functions/status.ts index 552fe6c07a..0b3ab5310a 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -1,3 +1,5 @@ +import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js"; + export type StatusLevel = "ok" | "info" | "warn" | "error"; export interface StatusProblem { @@ -58,6 +60,7 @@ export interface StatusInputs { }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; + indexPersistence?: IndexPersistenceStatus | null; } export interface StatusReport { @@ -72,6 +75,7 @@ export interface StatusReport { health: StatusInputs["health"]; provider: { llm: string; embeddings: string; circuitBreaker: StatusInputs["circuitBreaker"] }; index: StatusInputs["index"]; + indexPersistence: IndexPersistenceStatus | null; graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null; functions: Array; flags: StatusFlag[]; @@ -81,6 +85,7 @@ export interface StatusReport { const FAILURE_RATE_THRESHOLD = 0.2; const FAILURE_MIN_CALLS = 5; const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60; +const VECTOR_STRING_WARN_RATIO = 0.8; const LEVEL_RANK: Record = { ok: 0, info: 1, warn: 2, error: 3 }; const FUNCTION_FIXES: Record = { @@ -168,6 +173,43 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } + const persistence = input.indexPersistence ?? null; + if (persistence) { + const legs: Array<[string, IndexLegStatus | null]> = [ + ["BM25", persistence.bm25], + ["vector", persistence.vector], + ]; + for (const [label, leg] of legs) { + if (!leg) continue; + if (leg.lastError) { + problems.push({ + level: "error", + code: "index-save-failing", + message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`, + fix: "Check the server log for the failing state write. The next save retries automatically.", + }); + } + const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined); + if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { + problems.push({ + level: "warn", + code: "index-save-stale", + message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, + fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", + }); + } + } + const vectorChars = persistence.vector?.serializedChars ?? null; + if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) { + problems.push({ + level: "warn", + code: "vector-index-near-string-limit", + message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`, + fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.", + }); + } + } + let graph: StatusReport["graph"] = null; if (input.graph) { const ageSeconds = secondsBetween(input.now, input.graph.updatedAt); @@ -218,6 +260,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport { circuitBreaker: input.circuitBreaker, }, index: input.index, + indexPersistence: persistence, graph, functions, flags: input.flags, @@ -248,6 +291,29 @@ function row(label: string, value: string): string { return `${escapeHtml(label)}${value}`; } +function legSummary(report: StatusReport, leg: IndexLegStatus): string { + if (leg.lastError) return `failing: ${leg.lastError}`; + const saved = leg.lastSavedAt ? `saved ${formatDuration(secondsBetween(new Date(report.checkedAt), leg.lastSavedAt))} ago` : "not saved since start"; + return leg.dirtySince ? `${saved}, unsaved changes pending` : saved; +} + +function indexPersistenceRows(report: StatusReport): string { + const persistence = report.indexPersistence; + if (!persistence) return ""; + let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25))); + if (persistence.vector) { + rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector))); + const chars = persistence.vector.serializedChars; + if (chars !== null) { + rows += row( + "Vector index size", + escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`), + ); + } + } + return rows; +} + export function renderStatusHtml(report: StatusReport, styleNonce: string): string { const problems = report.problems.length ? report.problems @@ -330,6 +396,7 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off") ${row("Observations indexed", escapeHtml(idx.observationsIndexed))} ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))} ${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))} +${indexPersistenceRows(report)}

Knowledge graph

${graph diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 19b05cb598..28a24d5a1c 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -4,8 +4,9 @@ import type { StateKV } from "./kv.js"; import { KV, generateId } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; +import { getIndexSaveIntervalMs } from "../config.js"; -const DEBOUNCE_MS = 5000; +export const V8_MAX_STRING_CHARS = 536_870_888; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; const BM25_KEY = "data"; @@ -41,8 +42,31 @@ type IndexShardManifest = { type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; + saveIntervalMs?: number; + now?: () => number; }; +export type IndexLeg = "bm25" | "vector"; + +export interface IndexLegStatus { + lastSavedAt: string | null; + lastError: string | null; + lastErrorAt: string | null; + dirtySince: string | null; + serializedChars: number | null; +} + +export interface IndexPersistenceStatus { + saveIntervalMs: number; + saving: boolean; + bm25: IndexLegStatus; + vector: IndexLegStatus | null; +} + +function emptyLegStatus(): IndexLegStatus { + return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null }; +} + function shardChars(options: IndexPersistenceOptions): number { const configured = options.shardChars; if (typeof configured !== "number" || !Number.isFinite(configured)) { @@ -81,39 +105,72 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private running: Promise | null = null; + private queued: Promise | null = null; + private stopped = false; + private lastSaveAt: number; + private dirtyEpoch = 0; + private markedDuringRunAt: number | null = null; + private legs: Record = { + bm25: emptyLegStatus(), + vector: emptyLegStatus(), + }; + private readonly saveIntervalMs: number; + private readonly now: () => number; constructor( private kv: StateKV, private bm25: SearchIndex, private vector: VectorIndex | null, private options: IndexPersistenceOptions = {}, - ) {} + ) { + this.now = options.now ?? Date.now; + const interval = options.saveIntervalMs; + this.saveIntervalMs = + typeof interval === "number" && Number.isFinite(interval) && interval > 0 + ? interval + : getIndexSaveIntervalMs(); + this.lastSaveAt = this.now(); + } scheduleSave(): void { - if (this.timer) clearTimeout(this.timer); - // setTimeout discards the returned promise, so any rejection inside - // save() would surface as unhandledRejection and crash the process - // under sustained iii-engine write timeouts (issue #204). Funnel - // rejections through logFailure() instead. + if (this.stopped) return; + const now = this.now(); + this.dirtyEpoch++; + if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now; + for (const leg of this.activeLegs()) { + if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString(); + } + if (this.timer) return; + const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now); this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); - }, DEBOUNCE_MS); + this.timer = null; + this.save().catch((err) => this.logFailure("bm25", err)); + }, delay); } - async save(): Promise { - if (this.timer) { - clearTimeout(this.timer); - this.timer = null; - } - try { - await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { - await this.saveVectorIndex(this.vector.serialize()); - } - } catch (err) { - this.logFailure(err); + save(): Promise { + this.clearTimer(); + if (this.queued) return this.queued; + if (this.running) { + const queued = this.running.then(() => { + this.queued = null; + return this.startRun(); + }); + this.queued = queued; + return queued; } + return this.startRun(); + } + + status(): IndexPersistenceStatus { + return { + saveIntervalMs: this.saveIntervalMs, + saving: this.running !== null, + bm25: { ...this.legs.bm25 }, + vector: this.vector ? { ...this.legs.vector } : null, + }; } async load(): Promise<{ @@ -137,27 +194,80 @@ export class IndexPersistence { } stop(): void { + this.stopped = true; + this.clearTimer(); + } + + private clearTimer(): void { if (this.timer) { clearTimeout(this.timer); this.timer = null; } } - private logFailure(err: unknown): void { - const now = Date.now(); - // Throttle: persistence failures under load arrive in bursts - // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + private activeLegs(): IndexLeg[] { + return this.vector ? ["bm25", "vector"] : ["bm25"]; + } + + private startRun(): Promise { + const run = this.runSave().finally(() => { + if (this.running === run) this.running = null; + }); + this.running = run; + return run; + } + + private async runSave(): Promise { + this.markedDuringRunAt = null; + this.lastSaveAt = this.now(); + await this.saveLeg("bm25", async () => { + const serialized = this.bm25.serialize(); + this.legs.bm25.serializedChars = serialized.length; + await this.saveBm25Index(serialized); + }); + const vector = this.vector; + if (vector) { + await this.saveLeg("vector", async () => { + const serialized = vector.serialize(); + this.legs.vector.serializedChars = serialized.length; + await this.saveVectorIndex(serialized); + }); + } + } + + private async saveLeg(leg: IndexLeg, run: () => Promise): Promise { + const status = this.legs[leg]; + const epoch = this.dirtyEpoch; + try { + await run(); + status.lastSavedAt = new Date(this.now()).toISOString(); + status.lastError = null; + status.lastErrorAt = null; + if (this.dirtyEpoch === epoch) { + status.dirtySince = null; + } else if (this.markedDuringRunAt !== null) { + status.dirtySince = new Date(this.markedDuringRunAt).toISOString(); + } + } catch (err) { + status.lastError = errorMessage(err); + status.lastErrorAt = new Date(this.now()).toISOString(); + if (status.dirtySince === null) status.dirtySince = status.lastErrorAt; + this.logFailure(leg, err); + } + } + + private logFailure(leg: IndexLeg, err: unknown): void { + const now = this.now(); + if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(leg, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, { code, message, hint: code === "TIMEOUT" - ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next debounce flush" + ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save" : undefined, }); } diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d8a447f2e0..2fc2f3285a 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -12,7 +12,7 @@ import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js"; -import { findUnindexedObservations, getSearchIndex, getVectorIndex } from "../functions/search.js"; +import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js"; import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; @@ -373,6 +373,7 @@ export function registerApiTriggers( }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), + indexPersistence: getIndexPersistenceStatus(), }); const accept = req.headers?.["accept"] ?? req.headers?.["Accept"]; const format = req.query_params?.["format"]; diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts new file mode 100644 index 0000000000..54eff133b8 --- /dev/null +++ b/test/index-persistence-throttle.test.ts @@ -0,0 +1,335 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js"; +import { SearchIndex } from "../src/state/search-index.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js"; +import type { CompressedObservation } from "../src/types.js"; + +const BM25_SCOPE = "mem:index:bm25"; + +function mockKV() { + const store = new Map>(); + const sets: Array<{ scope: string; key: string }> = []; + return { + sets, + store, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + sets.push({ scope, key }); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function obs(id: string, title: string): CompressedObservation { + return { + id, + sessionId: "ses_1", + timestamp: new Date().toISOString(), + type: "file_edit", + title, + facts: [], + narrative: `${title} narrative`, + concepts: [], + files: [], + importance: 5, + }; +} + +function manifestSaves(kv: ReturnType, key: string): number { + return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length; +} + +describe("IndexPersistence save throttling", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("saves at most once per interval however often changes are scheduled", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(59_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + + await vi.advanceTimersByTimeAsync(1_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + for (let i = 0; i < 50; i++) persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + await vi.advanceTimersByTimeAsync(30_000); + expect(manifestSaves(kv, "data:manifest")).toBe(2); + }); + + it("an explicit save runs immediately and cancels the pending timer", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + + await vi.advanceTimersByTimeAsync(120_000); + expect(manifestSaves(kv, "data:manifest")).toBe(1); + }); + + it("never runs two saves at once and coalesces requests made during a save into one", async () => { + let release: () => void = () => undefined; + let inFlight = 0; + let maxInFlight = 0; + let gate: Promise | null = new Promise((resolve) => { + release = resolve; + }); + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + if (gate) await gate; + const result = await kv.set(scope, key, data); + inFlight--; + return result; + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const first = persistence.save(); + const second = persistence.save(); + const third = persistence.save(); + expect(second).toBe(third); + expect(persistence.status().saving).toBe(true); + + gate = null; + release(); + await Promise.all([first, second, third]); + + expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(maxInFlight).toBe(1); + expect(persistence.status().saving).toBe(false); + }); + + it("a failing BM25 leg does not stop the vector leg from saving", async () => { + const failingKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed"); + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + persistence.scheduleSave(); + await persistence.save(); + + const status = persistence.status(); + expect(status.bm25.lastError).toBe("bm25 manifest write failed"); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.lastError).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.serializedChars).toBeGreaterThan(0); + + const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + expect(loaded.bm25).toBeNull(); + expect(loaded.vector?.size).toBe(1); + }); + + it("keeps a leg dirty when a change arrives while it is being saved", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let held = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (!held) { + held = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + expect(persistence.status().bm25.lastSavedAt).not.toBeNull(); + expect(persistence.status().bm25.dirtySince).not.toBeNull(); + + await persistence.save(); + expect(persistence.status().bm25.dirtySince).toBeNull(); + }); + + it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => { + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let heldBm25Manifest = false; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) { + heldBm25Manifest = true; + await gate; + } + return kv.set(scope, key, data); + }, + }; + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const vector = new VectorIndex(); + vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 }); + + const saving = persistence.save(); + persistence.scheduleSave(); + release(); + await saving; + + const status = persistence.status(); + expect(status.bm25.dirtySince).not.toBeNull(); + expect(status.vector?.dirtySince).toBeNull(); + expect(status.vector?.lastSavedAt).not.toBeNull(); + }); + + it("stop prevents later scheduled saves", async () => { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_1", "alpha")); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 }); + persistence.stop(); + persistence.scheduleSave(); + await vi.advanceTimersByTimeAsync(10_000); + expect(manifestSaves(kv, "data:manifest")).toBe(0); + }); +}); + +function statusInputs(overrides: Partial = {}): StatusInputs { + return { + now: new Date("2026-09-25T12:00:00.000Z"), + version: "0.9.29", + engineVersion: "0.22.1", + uptimeSeconds: 100, + ports: { rest: 3111, streams: 3112, viewer: 3113 }, + health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" }, + circuitBreaker: null, + functionMetrics: [], + provider: "llm", + embeddingProvider: "embeddings", + flags: [], + index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 }, + graph: null, + graphExtractionEnabled: false, + ...overrides, + }; +} + +const cleanLeg = { + lastSavedAt: "2026-09-25T11:59:00.000Z", + lastError: null, + lastErrorAt: null, + dirtySince: null, + serializedChars: 1000, +}; + +describe("status reports index persistence", () => { + it("is quiet when both legs saved cleanly", () => { + const report = evaluateStatus( + statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }), + ); + expect(report.problems).toEqual([]); + expect(report.indexPersistence?.vector?.serializedChars).toBe(1000); + }); + + it("reports a failing leg as an error", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" }, + }, + }), + ); + expect(report.status).toBe("error"); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]); + expect(report.problems[0].message).toContain("vector"); + }); + + it("warns when the vector index passes 80% of the Node.js string limit", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: cleanLeg, + vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) }, + }, + }), + ); + expect(report.status).toBe("warn"); + expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]); + expect(report.problems[0].message).toContain("85%"); + }); + + it("warns when unsaved changes are older than twice the save interval", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" }, + vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" }, + }, + }), + ); + expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]); + expect(report.problems[0].message).toContain("BM25"); + }); + + it("shows save state and vector size on the status page", () => { + const report = evaluateStatus( + statusInputs({ + indexPersistence: { + saveIntervalMs: 600_000, + saving: false, + bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" }, + vector: { ...cleanLeg, serializedChars: 53_687_089 }, + }, + }), + ); + const html = renderStatusHtml(report, "n"); + expect(html).toContain(""); + expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)"); + }); +}); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index ba47c3ef72..802084bd2d 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -670,7 +670,7 @@ describe("IndexPersistence", () => { it("scheduleSave debounces multiple calls", async () => { const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null); + const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 }); persistence.scheduleSave(); persistence.scheduleSave(); @@ -721,7 +721,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); let unhandled = false; const onUnhandled = () => { @@ -751,7 +751,7 @@ describe("IndexPersistence", () => { }; const bm25 = new SearchIndex(); bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null); + const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); await expect(persistence.save()).resolves.toBeUndefined(); });
BM25 savesaved 1m ago, unsaved changes pending