Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/health/monitor.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { ISdk } from "iii-sdk";
import { getIndexPersistenceStatus } from "../state/index-persistence.js";
import type { HealthSnapshot } from "../types.js";
import type { StateKV } from "../state/kv.js";
import { KV } from "../state/schema.js";
Expand Down Expand Up @@ -80,6 +81,7 @@ export function registerHealthMonitor(
eventLoopLagMs,
uptimeSeconds: uptime,
kvConnectivity,
indexPersistence: getIndexPersistenceStatus(),
status: "healthy",
alerts: [],
};
Expand Down
17 changes: 17 additions & 0 deletions src/health/thresholds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,23 @@ export function evaluateHealth(
notes.push(`memory_heap_tight_${Math.round(memPercent)}%_rss${memMb}mb`);
}

const index = snapshot.indexPersistence;
if (index) {
if (index.lastError) {
alerts.push("index_persist_error");
degraded = true;
} else if (index.stale && index.dirtySince !== null) {
const staleMin = Math.round((Date.now() - index.dirtySince) / 60000);
alerts.push(`index_persist_stale_${staleMin}m`);
degraded = true;
} else {
const savedAt = index.lastSavedAt
? `${Math.round((Date.now() - new Date(index.lastSavedAt).getTime()) / 60000)}m_ago`
: "never";
notes.push(`index_docs_${index.bm25Docs}_vec_${index.vectors}_saved_${savedAt}`);
}
}

const status = critical ? "critical" : degraded ? "degraded" : "healthy";
return { status, alerts, notes };
}
52 changes: 52 additions & 0 deletions src/state/index-persistence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { logger } from "../logger.js";
import { safeAudit } from "../functions/audit.js";

const DEBOUNCE_MS = 5000;
const STALE_AFTER_MS = 60 * 60 * 1000;
const FAILURE_LOG_THROTTLE_MS = 60_000;
const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";
const BM25_KEY = "data";
Expand All @@ -24,6 +25,50 @@ type IndexShardManifest = {
chars: number;
};

/**
* What the index persistence layer knows about itself. A search index that
* stops being written is invisible from the outside: search keeps answering
* from memory and only a restart reveals that the last snapshot is old.
*/
export interface IndexPersistenceStatus {
bm25Docs: number;
vectors: number;
lastSavedAt: string | null;
lastError: string | null;
/** When the index first went dirty without a successful save since. */
dirtySince: number | null;
stale: boolean;
}

let status: IndexPersistenceStatus = {
bm25Docs: 0,
vectors: 0,
lastSavedAt: null,
lastError: null,
dirtySince: null,
stale: false,
};

/** Snapshot of index persistence health, for /agentmemory/health. */
export function getIndexPersistenceStatus(): IndexPersistenceStatus {
return {
...status,
stale: status.dirtySince !== null && Date.now() - status.dirtySince > STALE_AFTER_MS,
};
}

/** Test seam. */
export function resetIndexPersistenceStatus(): void {
status = {
bm25Docs: 0,
vectors: 0,
lastSavedAt: null,
lastError: null,
dirtySince: null,
stale: false,
};
}

type IndexPersistenceOptions = {
shardChars?: number;
createGeneration?: () => string;
Expand Down Expand Up @@ -77,6 +122,7 @@ export class IndexPersistence {
) {}

scheduleSave(): void {
if (status.dirtySince === null) status.dirtySince = Date.now();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

Keep dirty state for updates scheduled during an active save.

If scheduleSave() runs while an earlier save() awaits KV writes, the earlier save clears dirtySince at Line 150. The newer update can still be unsaved. Health then reports a clean index until the newer save completes.

Track a revision for each scheduled update. Clear dirtySince only if no later revision was scheduled.

Proposed fix
 let status: IndexPersistenceStatus = {
   // ...
 };
+let dirtyRevision = 0;

 export function resetIndexPersistenceStatus(): void {
   status = {
     // ...
   };
+  dirtyRevision = 0;
 }

 scheduleSave(): void {
   if (status.dirtySince === null) status.dirtySince = Date.now();
+  dirtyRevision += 1;
   // ...
 }

 async save(): Promise<void> {
+  const saveRevision = dirtyRevision;
   // ...
   try {
     // ...
     status.lastSavedAt = new Date().toISOString();
     status.lastError = null;
-    status.dirtySince = null;
+    if (dirtyRevision === saveRevision) status.dirtySince = null;

Also applies to: 146-150

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/state/index-persistence.ts` at line 125, Update scheduleSave() and save()
to track a monotonically increasing revision for each scheduled update,
capturing the revision when save begins and clearing dirtySince only when that
revision is still current after KV writes complete. Preserve dirtySince when a
newer update was scheduled during the active save so health continues to report
the index as dirty until the latest save finishes.

if (this.timer) clearTimeout(this.timer);
// setTimeout discards the returned promise, so any rejection inside
// save() would surface as unhandledRejection and crash the process
Expand All @@ -97,7 +143,13 @@ export class IndexPersistence {
if (this.vector) {
await this.saveVectorIndex(this.vector.serialize());
}
status.bm25Docs = this.bm25.size;
status.vectors = this.vector ? this.vector.size : 0;
status.lastSavedAt = new Date().toISOString();
status.lastError = null;
status.dirtySince = null;
} catch (err) {
status.lastError = err instanceof Error ? err.message : String(err);
this.logFailure(err);
}
}
Expand Down
9 changes: 9 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,15 @@ export interface HealthSnapshot {
eventLoopLagMs: number;
uptimeSeconds: number;
kvConnectivity?: { status: string; latencyMs?: number; error?: string };
/** Freshness of the persisted BM25/vector index, see getIndexPersistenceStatus(). */
indexPersistence?: {
bm25Docs: number;
vectors: number;
lastSavedAt: string | null;
lastError: string | null;
dirtySince: number | null;
stale: boolean;
};
status: "healthy" | "degraded" | "critical";
alerts: string[];
notes?: string[];
Expand Down
109 changes: 109 additions & 0 deletions test/index-freshness.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { describe, it, expect, vi, beforeEach } from "vitest";

vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));

import { evaluateHealth } from "../src/health/thresholds.js";
import type { HealthSnapshot } from "../src/types.js";

function snapshot(
indexPersistence?: HealthSnapshot["indexPersistence"],
): HealthSnapshot {
return {
connectionState: "connected",
workers: [],
memory: { heapUsed: 10, heapTotal: 100, rss: 10 * 1024 * 1024, external: 0 },
cpu: { userMicros: 0, systemMicros: 0, percent: 1 },
eventLoopLagMs: 1,
uptimeSeconds: 100,
kvConnectivity: { status: "ok", latencyMs: 1 },
indexPersistence,
status: "healthy",
alerts: [],
};
}

describe("search-index freshness in health", () => {
it("reports docs, vectors and save age as a note when the index is being written", () => {
const res = evaluateHealth(
snapshot({
bm25Docs: 12400,
vectors: 12400,
lastSavedAt: new Date(Date.now() - 5 * 60_000).toISOString(),
lastError: null,
dirtySince: null,
stale: false,
}),
);
expect(res.status).toBe("healthy");
expect(res.notes.join(" ")).toContain("index_docs_12400_vec_12400_saved_5m_ago");
});

it("degrades when the index has been dirty past the stale window", () => {
const res = evaluateHealth(
snapshot({
bm25Docs: 12400,
vectors: 12400,
lastSavedAt: new Date(Date.now() - 20 * 24 * 3600_000).toISOString(),
lastError: null,
dirtySince: Date.now() - 3 * 3600_000,
stale: true,
}),
);
expect(res.status).toBe("degraded");
expect(res.alerts.some((a) => a.startsWith("index_persist_stale_"))).toBe(true);
});

it("degrades when the last save failed", () => {
const res = evaluateHealth(
snapshot({
bm25Docs: 100,
vectors: 0,
lastSavedAt: null,
lastError: "TIMEOUT: invocation timed out after 180000ms",
dirtySince: Date.now(),
stale: false,
}),
);
expect(res.status).toBe("degraded");
expect(res.alerts).toContain("index_persist_error");
});

it("says nothing when the snapshot carries no index information", () => {
const res = evaluateHealth(snapshot(undefined));
expect(res.status).toBe("healthy");
expect(res.notes.join(" ")).not.toContain("index_");
});
});

describe("index persistence status tracking", () => {
beforeEach(() => {
vi.resetModules();
});

it("starts clean, marks dirty on scheduleSave and reports stale only past the window", async () => {
const { getIndexPersistenceStatus, resetIndexPersistenceStatus, IndexPersistence } =
await import("../src/state/index-persistence.js");
const { SearchIndex } = await import("../src/state/search-index.js");
resetIndexPersistenceStatus();

expect(getIndexPersistenceStatus().dirtySince).toBeNull();
expect(getIndexPersistenceStatus().stale).toBe(false);

const kv = {
get: async () => null,
set: async () => {},
delete: async () => {},
list: async () => [],
};
const persistence = new IndexPersistence(kv as never, new SearchIndex(), null);
persistence.scheduleSave();

const dirty = getIndexPersistenceStatus();
expect(dirty.dirtySince).not.toBeNull();
expect(dirty.stale).toBe(false); // just went dirty, well inside the window

persistence.stop();
});
});