diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index a09af90ff4..65240646c2 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -21,6 +21,10 @@ const PENDING_LOG_SAVE_THRESHOLD = 500; const EARLY_SAVE_MIN_GAP_MS = 5_000; const BACKFILL_MARKER_KEY = "vectors:backfill"; const MEMORY_ID_PREFIX = "mem_"; +const LEGACY_SHARD_SCOPE_PREFIXES = [`${KV.bm25Index}:bm25:`, `${KV.bm25Index}:vectors:`]; +const LEGACY_SHARD_KEY = "data"; +const LEGACY_SHARD_SUFFIX = /^((?:[0-9a-z]+-[0-9a-f]{32})|(?:idx_[0-9a-z]+_[0-9a-f]{12})):\d{5}$/; +const ORPHAN_SWEEP_KEY = "legacy:sweep"; function auditIndexPersistEnabled(): boolean { const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; @@ -159,6 +163,12 @@ function isValidShardDescriptor( ); } +function legacyShardGeneration(scope: string): string | null | undefined { + const prefix = LEGACY_SHARD_SCOPE_PREFIXES.find((candidate) => scope.startsWith(candidate)); + if (!prefix) return undefined; + return LEGACY_SHARD_SUFFIX.exec(scope.slice(prefix.length))?.[1] ?? null; +} + function isPersistedVector(row: unknown): row is PersistedVector { if (!row || typeof row !== "object") return false; const candidate = row as Partial; @@ -563,7 +573,7 @@ export class IndexPersistence { async load(): Promise { if (!this.vector) { - await this.removeLegacyBm25Snapshot(); + await this.finishLegacyCleanup(); return { vector: null, state: "none", savedAt: null }; } @@ -581,7 +591,7 @@ export class IndexPersistence { this.hasVectorCheckpoint = loaded.size > 0; this.checkpointBackfillSince = meta.backfillSince; await this.removeLegacyVectorSnapshotIfPresent(); - await this.removeLegacyBm25Snapshot(); + await this.finishLegacyCleanup(); const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined; this.vectorCountShortfall = expectedCount !== undefined && loaded.size < expectedCount @@ -788,7 +798,7 @@ export class IndexPersistence { if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null }; const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0); if (!legacyPresent) { - await this.removeLegacyBm25Snapshot(); + await this.finishLegacyCleanup(); return { vector: null, state: "none", savedAt: null }; } @@ -816,7 +826,7 @@ export class IndexPersistence { } await this.removeLegacyVectorSnapshot(manifest.value); - await this.removeLegacyBm25Snapshot(); + await this.finishLegacyCleanup(); await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], { vectors: legacy.size, buckets: this.bucketCountInUse(), @@ -860,6 +870,74 @@ export class IndexPersistence { if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup"); } + private async finishLegacyCleanup(): Promise { + await this.removeLegacyBm25Snapshot(); + await this.sweepOrphanedGenerations(); + } + + private async sweepOrphanedGenerations(): Promise { + try { + if ((await this.kv.get(KV.bm25Index, ORPHAN_SWEEP_KEY)) != null) return; + const candidates = new Map(); + for (const scope of await this.kv.listGroups()) { + const generation = legacyShardGeneration(scope); + if (generation === undefined) continue; + if (generation === null) { + logger.warn("index persistence: orphan sweep skipped, unrecognized legacy shard scope", { scope }); + return; + } + candidates.set(scope, generation); + } + const named = await this.namedLegacyGenerations(); + if (!named) return; + const orphans = [...candidates].filter(([, generation]) => !named.has(generation)); + const failures = await inBatches(orphans, WRITE_CONCURRENCY, async ([scope]) => { + if (!(await this.deleteKey(scope, LEGACY_SHARD_KEY, "orphan_generation_sweep"))) throw new Error(scope); + }); + const summary = { + generations: new Set(orphans.map(([, generation]) => generation)).size, + shards: orphans.length, + failed: failures.length, + }; + if (failures.length > 0) { + logger.warn("index persistence: orphan sweep left shards behind; the next boot retries", summary); + return; + } + if (candidates.size > orphans.length) { + if (orphans.length > 0) logger.info("index persistence: swept orphaned legacy index generations", summary); + return; + } + await this.kv.set(KV.bm25Index, ORPHAN_SWEEP_KEY, { v: 1, at: new Date(this.now()).toISOString(), ...summary }); + await this.auditIndexPersistence("sweep", [statePath(KV.bm25Index, ORPHAN_SWEEP_KEY)], summary); + logger.info("index persistence: swept orphaned legacy index generations", summary); + } catch (err) { + logger.warn("index persistence: orphan sweep failed; the next boot retries", { message: errorMessage(err) }); + } + } + + private async namedLegacyGenerations(): Promise | null> { + const named = new Set(); + for (const [key, label] of [[LEGACY_BM25_MANIFEST_KEY, "bm25"], [LEGACY_VECTOR_MANIFEST_KEY, "vector"]] as const) { + const read = await this.readIndexValue(KV.bm25Index, key, label, "manifest"); + if (!read.ok) return null; + const manifest = read.value; + if (manifest == null) continue; + const generations = typeof manifest === "object" && Array.isArray(manifest.shards) + ? manifest.shards + .filter((shard) => typeof shard?.scope === "string" && shard.scope.length > 0) + .map((shard) => legacyShardGeneration(shard.scope)) + : []; + if (typeof manifest === "object" && typeof manifest.generation === "string") generations.push(manifest.generation); + const usable = generations.filter((generation): generation is string => !!generation); + for (const generation of usable) named.add(generation); + if (usable.length === 0) { + logger.warn("index persistence: orphan sweep skipped, a legacy manifest names no generation", { key }); + return null; + } + } + return named; + } + private logFailure(err: unknown): void { const now = this.now(); if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; @@ -887,7 +965,7 @@ export class IndexPersistence { }); } - private async deleteKey(scope: string, key: string, reason: string): Promise { + private async deleteKey(scope: string, key: string, reason: string): Promise { let result = "deleted"; let error: string | undefined; try { @@ -903,10 +981,13 @@ export class IndexPersistence { result, error, }); + return error === undefined; } private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise { - await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason)); + await inBatches(shards, WRITE_CONCURRENCY, async (shard) => { + await this.deleteKey(shard.scope, shard.key, reason); + }); } private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise { diff --git a/src/state/kv.ts b/src/state/kv.ts index 7de13a2281..bfda4643a9 100644 --- a/src/state/kv.ts +++ b/src/state/kv.ts @@ -198,4 +198,16 @@ export class StateKV { if (this.backend !== 'redis' || !Array.isArray(values)) return values return orderLikeInsertion(values) } + + async listGroups(): Promise { + const result = await this.sdk.trigger, { groups?: unknown } | null>({ + function_id: 'state::list_groups', + payload: {}, + }) + const groups = result?.groups + if (!Array.isArray(groups) || !groups.every((group) => typeof group === 'string')) { + throw new Error('state::list_groups returned an unexpected shape') + } + return groups + } } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index e80327bde5..a795990e2b 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -29,6 +29,8 @@ function mockKV() { ops.push({ op: "list", scope }); return Array.from(store.get(scope)?.values() ?? []) as T[]; }, + listGroups: async (): Promise => + [...store].filter(([, entries]) => entries.size > 0).map(([scope]) => scope).sort(), }; } @@ -566,3 +568,209 @@ describe("index_persist audit gating", () => { expect(await indexPersistEntries()).toEqual([]); }); }); + +describe("IndexPersistence orphaned legacy generation sweep", () => { + const SWEEP_KEY = "legacy:sweep"; + const ORPHAN_IDX = "idx_mf000001_cccccccccccc"; + const ORPHAN_OLD = `mf000002-${"d".repeat(32)}`; + let kv: MockKV; + + beforeEach(() => { + kv = mockKV(); + }); + + async function writeOrphan(leg: "bm25" | "vectors", generation: string, shards = 2): Promise { + const scopes: string[] = []; + for (let i = 0; i < shards; i++) { + const scope = `${INDEX_SCOPE}:${leg}:${generation}:${String(i).padStart(5, "0")}`; + await kv.set(scope, "data", `orphan ${generation} ${i}`); + scopes.push(scope); + } + return scopes; + } + + async function writeBuckets(): Promise { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await new IndexPersistence(kv as never, vector, { bucketSize: 16 }).save(); + return vector; + } + + function present(scopes: string[]): boolean[] { + return scopes.map((scope) => kv.store.get(scope)?.has("data") ?? false); + } + + function load(vector: VectorIndex | null = new VectorIndex()) { + return new IndexPersistence(kv as never, vector, { bucketSize: 16 }).load(); + } + + it("removes generations no manifest names while migrating a legacy store", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); + await writeLegacyBm25Snapshot(kv); + const orphans = [...(await writeOrphan("bm25", ORPHAN_IDX)), ...(await writeOrphan("vectors", ORPHAN_OLD))]; + + const loaded = await load(); + + expect(loaded.state).toBe("migrated"); + expectSameVectors(loaded.vector, legacy); + expect(present(orphans)).toEqual([false, false, false, false]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ v: 1, generations: 2, shards: 4, failed: 0 }); + }); + + it("removes orphans from a store that already uses buckets and leaves the buckets alone", async () => { + const vector = await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + + const loaded = await load(); + + expect(loaded.state).toBe("buckets"); + expectSameVectors(loaded.vector, vector); + expect(present(orphans)).toEqual([false, false]); + expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ generations: 1, shards: 2 }); + }); + + it("keeps every generation a surviving manifest names", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); + const named = [...kv.store.keys()].filter((scope) => scope.includes(":vectors:")); + const orphans = await writeOrphan("vectors", ORPHAN_IDX); + + await load(null); + + expect(named.length).toBeGreaterThan(0); + expect(present(named).every(Boolean)).toBe(true); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); + expect(present(orphans)).toEqual([false, false]); + }); + + it("stays open while a manifest protects shards, and sweeps them once the manifest is gone", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); + const named = [...kv.store.keys()].filter((scope) => scope.includes(":vectors:")); + + await load(null); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + + await kv.delete(INDEX_SCOPE, "vectors:manifest"); + await load(null); + + expect(present(named).some(Boolean)).toBe(false); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).not.toBeNull(); + }); + + it("deletes nothing when a manifest read fails", async () => { + await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + const baseGet = kv.get; + kv.get = (async (scope: string, key: string) => { + if (key === "data:manifest") throw new Error("engine busy"); + return baseGet(scope, key); + }) as typeof kv.get; + + await load(); + + kv.get = baseGet; + expect(present(orphans)).toEqual([true, true]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + }); + + it("deletes nothing when a manifest names no generation", async () => { + await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, shards: "corrupt", chars: 0 }); + + await load(null); + + expect(present(orphans)).toEqual([true, true]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + }); + + it("deletes nothing when a legacy-prefixed scope does not parse", async () => { + await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + await kv.set(`${INDEX_SCOPE}:bm25:something-else`, "data", "unknown"); + + await load(); + + expect(present(orphans)).toEqual([true, true]); + expect(kv.store.get(`${INDEX_SCOPE}:bm25:something-else`)?.get("data")).toBe("unknown"); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + }); + + it("deletes nothing when scopes cannot be listed, and sweeps on the next boot", async () => { + await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + const baseListGroups = kv.listGroups; + kv.listGroups = async () => { + throw new Error("state::list_groups unavailable"); + }; + + await load(); + expect(present(orphans)).toEqual([true, true]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + + kv.listGroups = baseListGroups; + await load(); + expect(present(orphans)).toEqual([false, false]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).not.toBeNull(); + }); + + it("records no completion when a delete fails, and finishes on the next boot", async () => { + await writeBuckets(); + const orphans = await writeOrphan("bm25", ORPHAN_IDX); + const baseDelete = kv.delete; + kv.delete = async (scope: string, key: string) => { + if (scope === orphans[1]) throw new Error("delete timed out"); + return baseDelete(scope, key); + }; + + await load(); + expect(present(orphans)).toEqual([false, true]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull(); + + kv.delete = baseDelete; + await load(); + expect(present(orphans)).toEqual([false, false]); + expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ shards: 1, failed: 0 }); + }); + + it("runs once: a later boot neither lists scopes nor deletes", async () => { + await writeBuckets(); + await writeOrphan("bm25", ORPHAN_IDX); + await load(); + const later = await writeOrphan("vectors", ORPHAN_OLD); + let listed = 0; + const baseListGroups = kv.listGroups; + kv.listGroups = async () => { + listed++; + return baseListGroups(); + }; + const opsBefore = kv.ops.length; + + await load(); + + expect(listed).toBe(0); + expect(kv.ops.slice(opsBefore).filter((o) => o.op === "delete")).toEqual([]); + expect(present(later)).toEqual([true, true]); + }); + + it("audits each sweep delete and the summary when index auditing is on", async () => { + const previous = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = "1"; + try { + await writeBuckets(); + await writeOrphan("bm25", ORPHAN_IDX); + + await load(); + + const entries = await kv.list<{ operation: string; details: { action?: string; reason?: string } }>(currentAuditScope()); + const audited = entries.filter((entry) => entry.operation === "index_persist"); + expect(audited.filter((entry) => entry.details.reason === "orphan_generation_sweep")).toHaveLength(2); + expect(audited.some((entry) => entry.details.action === "sweep")).toBe(true); + } finally { + if (previous === undefined) delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + else process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previous; + } + }); +});