Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 87 additions & 6 deletions src/state/index-persistence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ const PENDING_LOG_SAVE_THRESHOLD = 500;
const EARLY_SAVE_MIN_GAP_MS = 5_000;
const BACKFILL_MARKER_KEY = "vectors:backfill";
const MEMORY_ID_PREFIX = "mem_";
const LEGACY_SHARD_SCOPE_PREFIXES = [`${KV.bm25Index}:bm25:`, `${KV.bm25Index}:vectors:`];
const LEGACY_SHARD_KEY = "data";
const LEGACY_SHARD_SUFFIX = /^((?:[0-9a-z]+-[0-9a-f]{32})|(?:idx_[0-9a-z]+_[0-9a-f]{12})):\d{5}$/;
const ORPHAN_SWEEP_KEY = "legacy:sweep";

function auditIndexPersistEnabled(): boolean {
const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
Expand Down Expand Up @@ -159,6 +163,12 @@ function isValidShardDescriptor(
);
}

function legacyShardGeneration(scope: string): string | null | undefined {
const prefix = LEGACY_SHARD_SCOPE_PREFIXES.find((candidate) => scope.startsWith(candidate));
if (!prefix) return undefined;
return LEGACY_SHARD_SUFFIX.exec(scope.slice(prefix.length))?.[1] ?? null;
}

function isPersistedVector(row: unknown): row is PersistedVector {
if (!row || typeof row !== "object") return false;
const candidate = row as Partial<PersistedVector>;
Expand Down Expand Up @@ -563,7 +573,7 @@ export class IndexPersistence {

async load(): Promise<VectorLoadResult> {
if (!this.vector) {
await this.removeLegacyBm25Snapshot();
await this.finishLegacyCleanup();
return { vector: null, state: "none", savedAt: null };
}

Expand All @@ -581,7 +591,7 @@ export class IndexPersistence {
this.hasVectorCheckpoint = loaded.size > 0;
this.checkpointBackfillSince = meta.backfillSince;
await this.removeLegacyVectorSnapshotIfPresent();
await this.removeLegacyBm25Snapshot();
await this.finishLegacyCleanup();
const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined;
this.vectorCountShortfall =
expectedCount !== undefined && loaded.size < expectedCount
Expand Down Expand Up @@ -788,7 +798,7 @@ export class IndexPersistence {
if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null };
const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0);
if (!legacyPresent) {
await this.removeLegacyBm25Snapshot();
await this.finishLegacyCleanup();
return { vector: null, state: "none", savedAt: null };
}

Expand Down Expand Up @@ -816,7 +826,7 @@ export class IndexPersistence {
}

await this.removeLegacyVectorSnapshot(manifest.value);
await this.removeLegacyBm25Snapshot();
await this.finishLegacyCleanup();
await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], {
vectors: legacy.size,
buckets: this.bucketCountInUse(),
Expand Down Expand Up @@ -860,6 +870,74 @@ export class IndexPersistence {
if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup");
}

private async finishLegacyCleanup(): Promise<void> {
await this.removeLegacyBm25Snapshot();
await this.sweepOrphanedGenerations();
}

private async sweepOrphanedGenerations(): Promise<void> {
try {
if ((await this.kv.get(KV.bm25Index, ORPHAN_SWEEP_KEY)) != null) return;
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
const candidates = new Map<string, string>();
for (const scope of await this.kv.listGroups()) {
const generation = legacyShardGeneration(scope);
if (generation === undefined) continue;
if (generation === null) {
logger.warn("index persistence: orphan sweep skipped, unrecognized legacy shard scope", { scope });
return;
}
candidates.set(scope, generation);
}
const named = await this.namedLegacyGenerations();
if (!named) return;
const orphans = [...candidates].filter(([, generation]) => !named.has(generation));
const failures = await inBatches(orphans, WRITE_CONCURRENCY, async ([scope]) => {
if (!(await this.deleteKey(scope, LEGACY_SHARD_KEY, "orphan_generation_sweep"))) throw new Error(scope);
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
});
const summary = {
generations: new Set(orphans.map(([, generation]) => generation)).size,
shards: orphans.length,
failed: failures.length,
};
if (failures.length > 0) {
logger.warn("index persistence: orphan sweep left shards behind; the next boot retries", summary);
return;
}
if (candidates.size > orphans.length) {
if (orphans.length > 0) logger.info("index persistence: swept orphaned legacy index generations", summary);
return;
}
await this.kv.set(KV.bm25Index, ORPHAN_SWEEP_KEY, { v: 1, at: new Date(this.now()).toISOString(), ...summary });
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
await this.auditIndexPersistence("sweep", [statePath(KV.bm25Index, ORPHAN_SWEEP_KEY)], summary);
logger.info("index persistence: swept orphaned legacy index generations", summary);
} catch (err) {
logger.warn("index persistence: orphan sweep failed; the next boot retries", { message: errorMessage(err) });
}
}

private async namedLegacyGenerations(): Promise<Set<string> | null> {
const named = new Set<string>();
for (const [key, label] of [[LEGACY_BM25_MANIFEST_KEY, "bm25"], [LEGACY_VECTOR_MANIFEST_KEY, "vector"]] as const) {
const read = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, key, label, "manifest");
if (!read.ok) return null;
const manifest = read.value;
if (manifest == null) continue;
const generations = typeof manifest === "object" && Array.isArray(manifest.shards)
? manifest.shards
.filter((shard) => typeof shard?.scope === "string" && shard.scope.length > 0)
.map((shard) => legacyShardGeneration(shard.scope))
: [];
if (typeof manifest === "object" && typeof manifest.generation === "string") generations.push(manifest.generation);
const usable = generations.filter((generation): generation is string => !!generation);
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
for (const generation of usable) named.add(generation);
if (usable.length === 0) {
logger.warn("index persistence: orphan sweep skipped, a legacy manifest names no generation", { key });
return null;
}
}
return named;
}

private logFailure(err: unknown): void {
const now = this.now();
if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return;
Expand Down Expand Up @@ -887,7 +965,7 @@ export class IndexPersistence {
});
}

private async deleteKey(scope: string, key: string, reason: string): Promise<void> {
private async deleteKey(scope: string, key: string, reason: string): Promise<boolean> {
let result = "deleted";
let error: string | undefined;
try {
Expand All @@ -903,10 +981,13 @@ export class IndexPersistence {
result,
error,
});
return error === undefined;
}

private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise<void> {
await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason));
await inBatches(shards, WRITE_CONCURRENCY, async (shard) => {
await this.deleteKey(shard.scope, shard.key, reason);
});
}

private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise<string | null> {
Expand Down
12 changes: 12 additions & 0 deletions src/state/kv.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,4 +198,16 @@ export class StateKV {
if (this.backend !== 'redis' || !Array.isArray(values)) return values
return orderLikeInsertion(values)
}

async listGroups(): Promise<string[]> {
const result = await this.sdk.trigger<Record<string, never>, { groups?: unknown } | null>({
function_id: 'state::list_groups',
payload: {},
})
const groups = result?.groups
if (!Array.isArray(groups) || !groups.every((group) => typeof group === 'string')) {
throw new Error('state::list_groups returned an unexpected shape')
}
return groups
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
}
}
208 changes: 208 additions & 0 deletions test/index-persistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ function mockKV() {
ops.push({ op: "list", scope });
return Array.from(store.get(scope)?.values() ?? []) as T[];
},
listGroups: async (): Promise<string[]> =>
[...store].filter(([, entries]) => entries.size > 0).map(([scope]) => scope).sort(),
};
}

Expand Down Expand Up @@ -566,3 +568,209 @@ describe("index_persist audit gating", () => {
expect(await indexPersistEntries()).toEqual([]);
});
});

describe("IndexPersistence orphaned legacy generation sweep", () => {
const SWEEP_KEY = "legacy:sweep";
const ORPHAN_IDX = "idx_mf000001_cccccccccccc";
const ORPHAN_OLD = `mf000002-${"d".repeat(32)}`;
let kv: MockKV;

beforeEach(() => {
kv = mockKV();
});

async function writeOrphan(leg: "bm25" | "vectors", generation: string, shards = 2): Promise<string[]> {
const scopes: string[] = [];
for (let i = 0; i < shards; i++) {
const scope = `${INDEX_SCOPE}:${leg}:${generation}:${String(i).padStart(5, "0")}`;
await kv.set(scope, "data", `orphan ${generation} ${i}`);
scopes.push(scope);
}
return scopes;
}

async function writeBuckets(): Promise<VectorIndex> {
const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
await new IndexPersistence(kv as never, vector, { bucketSize: 16 }).save();
return vector;
}

function present(scopes: string[]): boolean[] {
return scopes.map((scope) => kv.store.get(scope)?.has("data") ?? false);
}

function load(vector: VectorIndex | null = new VectorIndex()) {
return new IndexPersistence(kv as never, vector, { bucketSize: 16 }).load();
}

it("removes generations no manifest names while migrating a legacy store", async () => {
const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" });
await writeLegacyBm25Snapshot(kv);
const orphans = [...(await writeOrphan("bm25", ORPHAN_IDX)), ...(await writeOrphan("vectors", ORPHAN_OLD))];

const loaded = await load();

expect(loaded.state).toBe("migrated");
expectSameVectors(loaded.vector, legacy);
expect(present(orphans)).toEqual([false, false, false, false]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ v: 1, generations: 2, shards: 4, failed: 0 });
});

it("removes orphans from a store that already uses buckets and leaves the buckets alone", async () => {
const vector = await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);

const loaded = await load();

expect(loaded.state).toBe("buckets");
expectSameVectors(loaded.vector, vector);
expect(present(orphans)).toEqual([false, false]);
expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ generations: 1, shards: 2 });
});

it("keeps every generation a surviving manifest names", async () => {
const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" });
const named = [...kv.store.keys()].filter((scope) => scope.includes(":vectors:"));
const orphans = await writeOrphan("vectors", ORPHAN_IDX);

await load(null);

expect(named.length).toBeGreaterThan(0);
expect(present(named).every(Boolean)).toBe(true);
expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();
expect(present(orphans)).toEqual([false, false]);
});

it("stays open while a manifest protects shards, and sweeps them once the manifest is gone", async () => {
const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" });
const named = [...kv.store.keys()].filter((scope) => scope.includes(":vectors:"));

await load(null);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();

await kv.delete(INDEX_SCOPE, "vectors:manifest");
await load(null);

expect(present(named).some(Boolean)).toBe(false);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).not.toBeNull();
});

it("deletes nothing when a manifest read fails", async () => {
await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);
const baseGet = kv.get;
kv.get = (async <T>(scope: string, key: string) => {
if (key === "data:manifest") throw new Error("engine busy");
return baseGet<T>(scope, key);
}) as typeof kv.get;

await load();

kv.get = baseGet;
expect(present(orphans)).toEqual([true, true]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();
});

it("deletes nothing when a manifest names no generation", async () => {
await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);
await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, shards: "corrupt", chars: 0 });

await load(null);

expect(present(orphans)).toEqual([true, true]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();
});

it("deletes nothing when a legacy-prefixed scope does not parse", async () => {
await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);
await kv.set(`${INDEX_SCOPE}:bm25:something-else`, "data", "unknown");

await load();

expect(present(orphans)).toEqual([true, true]);
expect(kv.store.get(`${INDEX_SCOPE}:bm25:something-else`)?.get("data")).toBe("unknown");
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();
});

it("deletes nothing when scopes cannot be listed, and sweeps on the next boot", async () => {
await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);
const baseListGroups = kv.listGroups;
kv.listGroups = async () => {
throw new Error("state::list_groups unavailable");
};

await load();
expect(present(orphans)).toEqual([true, true]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();

kv.listGroups = baseListGroups;
await load();
expect(present(orphans)).toEqual([false, false]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).not.toBeNull();
});

it("records no completion when a delete fails, and finishes on the next boot", async () => {
await writeBuckets();
const orphans = await writeOrphan("bm25", ORPHAN_IDX);
const baseDelete = kv.delete;
kv.delete = async (scope: string, key: string) => {
if (scope === orphans[1]) throw new Error("delete timed out");
return baseDelete(scope, key);
};

await load();
expect(present(orphans)).toEqual([false, true]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toBeNull();

kv.delete = baseDelete;
await load();
expect(present(orphans)).toEqual([false, false]);
expect(await kv.get(INDEX_SCOPE, SWEEP_KEY)).toMatchObject({ shards: 1, failed: 0 });
});

it("runs once: a later boot neither lists scopes nor deletes", async () => {
await writeBuckets();
await writeOrphan("bm25", ORPHAN_IDX);
await load();
const later = await writeOrphan("vectors", ORPHAN_OLD);
let listed = 0;
const baseListGroups = kv.listGroups;
kv.listGroups = async () => {
listed++;
return baseListGroups();
};
const opsBefore = kv.ops.length;

await load();

expect(listed).toBe(0);
expect(kv.ops.slice(opsBefore).filter((o) => o.op === "delete")).toEqual([]);
expect(present(later)).toEqual([true, true]);
});

it("audits each sweep delete and the summary when index auditing is on", async () => {
const previous = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = "1";
try {
await writeBuckets();
await writeOrphan("bm25", ORPHAN_IDX);

await load();

const entries = await kv.list<{ operation: string; details: { action?: string; reason?: string } }>(currentAuditScope());
const audited = entries.filter((entry) => entry.operation === "index_persist");
expect(audited.filter((entry) => entry.details.reason === "orphan_generation_sweep")).toHaveLength(2);
expect(audited.some((entry) => entry.details.action === "sweep")).toBe(true);
} finally {
if (previous === undefined) delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
else process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previous;
}
});
});
Loading