From 49cf34f6a2fa10a6ffd3965fe313e3054d1cb0ed Mon Sep 17 00:00:00 2001 From: sbynode-ux Date: Tue, 11 Aug 2026 21:28:34 -0500 Subject: [PATCH] feat(health): surface search-index freshness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A search index that stops being persisted is invisible from the outside. Search keeps answering from the in-memory copy, /agentmemory/health keeps saying healthy, and the gap only appears on the next restart — when the server loads whatever snapshot last succeeded. On one install here that snapshot was 20 days old: 4944 docs restored against a corpus of 12801, so 61% of memory had quietly stopped being searchable, and the only trace was a throttled warn line. IndexPersistence now tracks its own outcome — doc and vector counts, last successful save, last error, and how long the index has been dirty without a save landing — and exposes it through getIndexPersistenceStatus(). The health snapshot carries it, so evaluateHealth can say so: notes: index_docs_12400_vec_12400_saved_5m_ago alerts: index_persist_error | index_persist_stale_m (both degrade) Nothing changes for a snapshot that carries no index information, so callers constructing HealthSnapshot themselves are unaffected. Signed-off-by: sbynode-ux --- src/health/monitor.ts | 2 + src/health/thresholds.ts | 17 +++++ src/state/index-persistence.ts | 52 ++++++++++++++++ src/types.ts | 9 +++ test/index-freshness.test.ts | 109 +++++++++++++++++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100644 test/index-freshness.test.ts diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 953c94a0f4..06c560dac6 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -1,4 +1,5 @@ import type { ISdk } from "iii-sdk"; +import { getIndexPersistenceStatus } from "../state/index-persistence.js"; import type { HealthSnapshot } from "../types.js"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; @@ -80,6 +81,7 @@ export function registerHealthMonitor( eventLoopLagMs, uptimeSeconds: uptime, kvConnectivity, + indexPersistence: getIndexPersistenceStatus(), status: "healthy", alerts: [], }; diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad16..c204961d91 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -76,6 +76,23 @@ export function evaluateHealth( notes.push(`memory_heap_tight_${Math.round(memPercent)}%_rss${memMb}mb`); } + const index = snapshot.indexPersistence; + if (index) { + if (index.lastError) { + alerts.push("index_persist_error"); + degraded = true; + } else if (index.stale && index.dirtySince !== null) { + const staleMin = Math.round((Date.now() - index.dirtySince) / 60000); + alerts.push(`index_persist_stale_${staleMin}m`); + degraded = true; + } else { + const savedAt = index.lastSavedAt + ? `${Math.round((Date.now() - new Date(index.lastSavedAt).getTime()) / 60000)}m_ago` + : "never"; + notes.push(`index_docs_${index.bm25Docs}_vec_${index.vectors}_saved_${savedAt}`); + } + } + const status = critical ? "critical" : degraded ? "degraded" : "healthy"; return { status, alerts, notes }; } diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..df9fab3f90 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -6,6 +6,7 @@ import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; const DEBOUNCE_MS = 5000; +const STALE_AFTER_MS = 60 * 60 * 1000; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; const BM25_KEY = "data"; @@ -24,6 +25,50 @@ type IndexShardManifest = { chars: number; }; +/** + * What the index persistence layer knows about itself. A search index that + * stops being written is invisible from the outside: search keeps answering + * from memory and only a restart reveals that the last snapshot is old. + */ +export interface IndexPersistenceStatus { + bm25Docs: number; + vectors: number; + lastSavedAt: string | null; + lastError: string | null; + /** When the index first went dirty without a successful save since. */ + dirtySince: number | null; + stale: boolean; +} + +let status: IndexPersistenceStatus = { + bm25Docs: 0, + vectors: 0, + lastSavedAt: null, + lastError: null, + dirtySince: null, + stale: false, +}; + +/** Snapshot of index persistence health, for /agentmemory/health. */ +export function getIndexPersistenceStatus(): IndexPersistenceStatus { + return { + ...status, + stale: status.dirtySince !== null && Date.now() - status.dirtySince > STALE_AFTER_MS, + }; +} + +/** Test seam. */ +export function resetIndexPersistenceStatus(): void { + status = { + bm25Docs: 0, + vectors: 0, + lastSavedAt: null, + lastError: null, + dirtySince: null, + stale: false, + }; +} + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; @@ -77,6 +122,7 @@ export class IndexPersistence { ) {} scheduleSave(): void { + if (status.dirtySince === null) status.dirtySince = Date.now(); if (this.timer) clearTimeout(this.timer); // setTimeout discards the returned promise, so any rejection inside // save() would surface as unhandledRejection and crash the process @@ -97,7 +143,13 @@ export class IndexPersistence { if (this.vector) { await this.saveVectorIndex(this.vector.serialize()); } + status.bm25Docs = this.bm25.size; + status.vectors = this.vector ? this.vector.size : 0; + status.lastSavedAt = new Date().toISOString(); + status.lastError = null; + status.dirtySince = null; } catch (err) { + status.lastError = err instanceof Error ? err.message : String(err); this.logFailure(err); } } diff --git a/src/types.ts b/src/types.ts index 2f3f0285f1..a8c8d0ccf1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -211,6 +211,15 @@ export interface HealthSnapshot { eventLoopLagMs: number; uptimeSeconds: number; kvConnectivity?: { status: string; latencyMs?: number; error?: string }; + /** Freshness of the persisted BM25/vector index, see getIndexPersistenceStatus(). */ + indexPersistence?: { + bm25Docs: number; + vectors: number; + lastSavedAt: string | null; + lastError: string | null; + dirtySince: number | null; + stale: boolean; + }; status: "healthy" | "degraded" | "critical"; alerts: string[]; notes?: string[]; diff --git a/test/index-freshness.test.ts b/test/index-freshness.test.ts new file mode 100644 index 0000000000..4f610d4d2e --- /dev/null +++ b/test/index-freshness.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { evaluateHealth } from "../src/health/thresholds.js"; +import type { HealthSnapshot } from "../src/types.js"; + +function snapshot( + indexPersistence?: HealthSnapshot["indexPersistence"], +): HealthSnapshot { + return { + connectionState: "connected", + workers: [], + memory: { heapUsed: 10, heapTotal: 100, rss: 10 * 1024 * 1024, external: 0 }, + cpu: { userMicros: 0, systemMicros: 0, percent: 1 }, + eventLoopLagMs: 1, + uptimeSeconds: 100, + kvConnectivity: { status: "ok", latencyMs: 1 }, + indexPersistence, + status: "healthy", + alerts: [], + }; +} + +describe("search-index freshness in health", () => { + it("reports docs, vectors and save age as a note when the index is being written", () => { + const res = evaluateHealth( + snapshot({ + bm25Docs: 12400, + vectors: 12400, + lastSavedAt: new Date(Date.now() - 5 * 60_000).toISOString(), + lastError: null, + dirtySince: null, + stale: false, + }), + ); + expect(res.status).toBe("healthy"); + expect(res.notes.join(" ")).toContain("index_docs_12400_vec_12400_saved_5m_ago"); + }); + + it("degrades when the index has been dirty past the stale window", () => { + const res = evaluateHealth( + snapshot({ + bm25Docs: 12400, + vectors: 12400, + lastSavedAt: new Date(Date.now() - 20 * 24 * 3600_000).toISOString(), + lastError: null, + dirtySince: Date.now() - 3 * 3600_000, + stale: true, + }), + ); + expect(res.status).toBe("degraded"); + expect(res.alerts.some((a) => a.startsWith("index_persist_stale_"))).toBe(true); + }); + + it("degrades when the last save failed", () => { + const res = evaluateHealth( + snapshot({ + bm25Docs: 100, + vectors: 0, + lastSavedAt: null, + lastError: "TIMEOUT: invocation timed out after 180000ms", + dirtySince: Date.now(), + stale: false, + }), + ); + expect(res.status).toBe("degraded"); + expect(res.alerts).toContain("index_persist_error"); + }); + + it("says nothing when the snapshot carries no index information", () => { + const res = evaluateHealth(snapshot(undefined)); + expect(res.status).toBe("healthy"); + expect(res.notes.join(" ")).not.toContain("index_"); + }); +}); + +describe("index persistence status tracking", () => { + beforeEach(() => { + vi.resetModules(); + }); + + it("starts clean, marks dirty on scheduleSave and reports stale only past the window", async () => { + const { getIndexPersistenceStatus, resetIndexPersistenceStatus, IndexPersistence } = + await import("../src/state/index-persistence.js"); + const { SearchIndex } = await import("../src/state/search-index.js"); + resetIndexPersistenceStatus(); + + expect(getIndexPersistenceStatus().dirtySince).toBeNull(); + expect(getIndexPersistenceStatus().stale).toBe(false); + + const kv = { + get: async () => null, + set: async () => {}, + delete: async () => {}, + list: async () => [], + }; + const persistence = new IndexPersistence(kv as never, new SearchIndex(), null); + persistence.scheduleSave(); + + const dirty = getIndexPersistenceStatus(); + expect(dirty.dirtySince).not.toBeNull(); + expect(dirty.stale).toBe(false); // just went dirty, well inside the window + + persistence.stop(); + }); +});