diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts index 3e7543abaf..42ee095fbd 100644 --- a/src/functions/auto-forget.ts +++ b/src/functions/auto-forget.ts @@ -3,7 +3,7 @@ import { markCaptureEventDeleted } from "../capture/event-record.js"; import type { Memory, CompressedObservation, Session } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV, collectAll } from "../state/kv.js"; -import { supersededAt } from "../state/memory-utils.js"; +import { deleteMemoryUnderLock, supersededAt } from "../state/memory-utils.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { unindexObservationSession } from "../state/obs-index.js"; import { recordAudit } from "./audit.js"; @@ -31,7 +31,7 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void { async (data: { dryRun?: boolean }): Promise => { const dryRun = data?.dryRun ?? false; const now = Date.now(); - const { decrementImageRef } = await import("./image-refs.js"); + const { decrementImageRef, releaseImageRef } = await import("./image-refs.js"); const result: AutoForgetResult = { ttlExpired: [], @@ -51,10 +51,9 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void { result.ttlExpired.push(mem.id); deletedIds.add(mem.id); if (!dryRun) { - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } - await kv.delete(KV.memories, mem.id); + const removed = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!removed) continue; + if (removed.imageRef) await releaseImageRef(kv, sdk, mem.id, removed.imageRef); facetIds.add(mem.id); await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], { resource: "memory", diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 33600d3cb0..7c8d0d5c62 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -10,6 +10,7 @@ import type { import { KV } from "../state/schema.js"; import { StateKV, collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import { removeSessionFromProjectIndex } from "../state/session-index.js"; import { unindexObservationSession } from "../state/obs-index.js"; import { isConsolidationEnabled } from "../config.js"; @@ -115,7 +116,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction("mem::evict", async (data: { dryRun?: boolean }): Promise => { const dryRun = data?.dryRun ?? false; - const { decrementImageRef } = await import("./image-refs.js"); + const { releaseImageRef } = await import("./image-refs.js"); let indexCleaned = false; const removeObservation = ( sessionId: string, @@ -271,8 +272,8 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { continue; } await unindexObservationSession(kv, o.id).catch(() => {}); - if (removed.imageData) await decrementImageRef(kv, sdk, removed.imageData); - if (removed.imageRef && removed.imageRef !== removed.imageData) await decrementImageRef(kv, sdk, removed.imageRef); + if (removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageData); + if (removed.imageRef && removed.imageRef !== removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageRef); await recordAudit(kv, "delete", "mem::evict", [o.id], { resource: "observation", reason: "low_importance_old_observation", @@ -319,8 +320,8 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { continue; } await unindexObservationSession(kv, o.id).catch(() => {}); - if (removed.imageData) await decrementImageRef(kv, sdk, removed.imageData); - if (removed.imageRef && removed.imageRef !== removed.imageData) await decrementImageRef(kv, sdk, removed.imageRef); + if (removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageData); + if (removed.imageRef && removed.imageRef !== removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageRef); await recordAudit(kv, "delete", "mem::evict", [o.id], { resource: "observation", reason: "project_observation_cap", @@ -344,8 +345,10 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { stats.expiredMemories++; evictedMemIds.add(mem.id); } else { + let deleted: Memory | null; try { - await kv.delete(KV.memories, mem.id); + deleted = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!deleted) continue; deletedMemIds.add(mem.id); stats.expiredMemories++; evictedMemIds.add(mem.id); @@ -361,9 +364,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { getSearchIndex().remove(mem.id); vectorIndexRemove(mem.id); indexCleaned = true; - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } + if (deleted.imageRef) await releaseImageRef(kv, sdk, mem.id, deleted.imageRef); await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "expired_memory", @@ -384,8 +385,10 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { if (dryRun) { stats.nonLatestMemories++; } else { + let deleted: Memory | null; try { - await kv.delete(KV.memories, mem.id); + deleted = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!deleted) continue; deletedMemIds.add(mem.id); stats.nonLatestMemories++; } catch (err) { @@ -400,9 +403,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { getSearchIndex().remove(mem.id); vectorIndexRemove(mem.id); indexCleaned = true; - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } + if (deleted.imageRef) await releaseImageRef(kv, sdk, mem.id, deleted.imageRef); await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "old_non_latest_memory", diff --git a/src/functions/governance.ts b/src/functions/governance.ts index 41ac200c62..e6b123f536 100644 --- a/src/functions/governance.ts +++ b/src/functions/governance.ts @@ -3,6 +3,7 @@ import type { Memory, GovernanceFilter, AuditEntry } from "../types.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { collectAll } from "../state/kv.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import { recordAudit, safeAudit, queryAudit } from "./audit.js"; import { deleteAccessLog } from "./access-tracker.js"; import { deleteFacetsForTargets } from "./facets.js"; @@ -21,9 +22,7 @@ async function deleteMemoryRecord( kv: StateKV, id: string, ): Promise<"deleted" | "missing"> { - const mem = await kv.get(KV.memories, id); - if (!mem) return "missing"; - await kv.delete(KV.memories, id); + if (!(await deleteMemoryUnderLock(kv, KV.memories, id))) return "missing"; await deleteAccessLog(kv, id); getSearchIndex().remove(id); vectorIndexRemove(id); diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.ts index 918a7e7c46..75012f0448 100644 --- a/src/functions/image-refs.ts +++ b/src/functions/image-refs.ts @@ -3,6 +3,7 @@ import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { deleteImage, touchImage } from "../utils/image-store.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { logger } from "../logger.js"; export async function getImageRefCount(kv: StateKV, filePath: string): Promise { const count = await kv.get(KV.imageRefs, filePath); @@ -36,3 +37,15 @@ export async function decrementImageRef(kv: StateKV, sdk: IIIClient, filePath: s } }); } + +export async function releaseImageRef(kv: StateKV, sdk: IIIClient, ownerId: string, filePath: string): Promise { + try { + await decrementImageRef(kv, sdk, filePath); + } catch (err) { + logger.warn("Image reference release failed after delete", { + ownerId, + imageRef: filePath, + error: err instanceof Error ? err.message : String(err), + }); + } +} diff --git a/src/functions/retention.ts b/src/functions/retention.ts index 97c05db910..a40e5f4a78 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -8,6 +8,7 @@ import type { import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { collectAll } from "../state/kv.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import type { AccessLog } from "./access-tracker.js"; import { emptyAccessLog, @@ -306,7 +307,7 @@ export function registerRetentionFunctions( ? data.maxEvict : 50; const maxEvict = Math.min(1000, Math.max(0, maxEvictRaw)); - const { decrementImageRef } = await import("./image-refs.js"); + const { releaseImageRef } = await import("./image-refs.js"); const allScores = await kv.list(KV.retentionScores); const candidates = allScores @@ -373,10 +374,12 @@ export function registerRetentionFunctions( continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + const removed = await deleteMemoryUnderLock(kv, scope, candidate.memoryId); + if (!removed) { + missingIds.push(candidate.memoryId); + continue; } - await kv.delete(scope, candidate.memoryId); + if (removed.imageRef) await releaseImageRef(kv, sdk, candidate.memoryId, removed.imageRef); evictedIds.push(candidate.memoryId); evicted++; if (resolvedSource === "semantic") evictedSemantic++; @@ -390,6 +393,12 @@ export function registerRetentionFunctions( } } + const orphanScoreIds: string[] = []; + for (const id of missingIds) { + if (await kv.get(KV.memories, id) || await kv.get(KV.semantic, id)) continue; + await kv.delete(KV.retentionScores, id); + orphanScoreIds.push(id); + } await deleteFacetsForTargets(kv, [...evictedIds, ...missingIds]); // Retention eviction is a structural delete path that removes @@ -408,6 +417,13 @@ export function registerRetentionFunctions( reason: "retention score below threshold", }); } + if (orphanScoreIds.length > 0) { + await recordAudit(kv, "delete", "mem::retention-evict", orphanScoreIds, { + resource: "retention_score", + removed: orphanScoreIds.length, + reason: "retention score for a memory that no longer exists", + }); + } logger.info("Retention-based eviction complete", { evicted, diff --git a/src/state/keyed-mutex.ts b/src/state/keyed-mutex.ts index 6ac2ac6ad7..308a5e7282 100644 --- a/src/state/keyed-mutex.ts +++ b/src/state/keyed-mutex.ts @@ -58,6 +58,10 @@ export function withKeyedLock( ); } +export function withReentrantKeyedLock(key: string, fn: () => Promise): Promise { + return isKeyedLockHeld(key) ? fn() : withKeyedLock(key, fn); +} + export function withKeyedLocks(keys: string[], fn: () => Promise): Promise { const ordered = [...new Set(keys)].sort(); const acquire = (index: number): Promise => index === ordered.length diff --git a/src/state/memory-utils.ts b/src/state/memory-utils.ts index 8c7cbe329e..86a463efa2 100644 --- a/src/state/memory-utils.ts +++ b/src/state/memory-utils.ts @@ -1,7 +1,22 @@ import type { CompressedObservation, Lesson, Memory } from "../types.js"; +import type { StateKV } from "./kv.js"; +import { withReentrantKeyedLock } from "./keyed-mutex.js"; const MAX_DATE_MS = 8.64e15; +export function deleteMemoryUnderLock( + kv: StateKV, + scope: string, + id: string, +): Promise { + return withReentrantKeyedLock("mem:remember", async () => { + const current = await kv.get(scope, id); + if (!current) return null; + await kv.delete(scope, id); + return current; + }); +} + export function supersededAt(previous: string | undefined, now: string): string { const prior = previous ? Date.parse(previous) : Number.NaN; const current = Date.parse(now); diff --git a/test/image-ref-release.test.ts b/test/image-ref-release.test.ts new file mode 100644 index 0000000000..7f9784231b --- /dev/null +++ b/test/image-ref-release.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { warn } = vi.hoisted(() => ({ warn: vi.fn() })); +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn, error: vi.fn() }, +})); + +import { releaseImageRef } from "../src/functions/image-refs.js"; +import { KV } from "../src/state/schema.js"; + +function kvWithFailingWrite(count: number) { + return { + get: async (scope: string) => (scope === KV.imageRefs ? count : null), + set: async () => { + throw new Error("image ref write failed"); + }, + delete: async () => {}, + list: async () => [], + }; +} + +describe("releaseImageRef", () => { + beforeEach(() => warn.mockClear()); + + it("logs a failed image ref write and does not throw", async () => { + const sdk = { trigger: vi.fn() }; + await expect(releaseImageRef(kvWithFailingWrite(2) as never, sdk as never, "mem_1", "/tmp/img-a.png")).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith("Image reference release failed after delete", { + ownerId: "mem_1", + imageRef: "/tmp/img-a.png", + error: "image ref write failed", + }); + }); + + it("decrements the count when the write succeeds", async () => { + const writes: Array<[string, string, unknown]> = []; + const kv = { + get: async () => 3, + set: async (scope: string, key: string, value: unknown) => { + writes.push([scope, key, value]); + return value; + }, + delete: async () => {}, + list: async () => [], + }; + await releaseImageRef(kv as never, { trigger: vi.fn() } as never, "mem_2", "/tmp/img-b.png"); + expect(writes).toEqual([[KV.imageRefs, "/tmp/img-b.png", 2]]); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/test/memory-delete-lock.test.ts b/test/memory-delete-lock.test.ts new file mode 100644 index 0000000000..66fd6d6cdc --- /dev/null +++ b/test/memory-delete-lock.test.ts @@ -0,0 +1,388 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/config.js", async (importOriginal) => ({ + ...(await importOriginal()), + isConsolidationEnabled: () => false, +})); + +const imageReleases: Array<{ path: string; lockHeld: boolean }> = []; +const imageFailures = new Set(); +vi.mock("../src/functions/image-refs.js", async (importOriginal) => { + const actual = await importOriginal(); + const decrementImageRef = vi.fn(async (_kv: unknown, _sdk: unknown, path: string) => { + const { isKeyedLockHeld } = await import("../src/state/keyed-mutex.js"); + imageReleases.push({ path, lockHeld: isKeyedLockHeld("mem:remember") }); + if (imageFailures.has(path)) throw new Error("image ref write failed"); + }); + const releaseImageRef = vi.fn(async (kv: unknown, sdk: unknown, ownerId: string, path: string) => { + try { + await decrementImageRef(kv, sdk, path); + } catch (err) { + const { logger } = await import("../src/logger.js"); + logger.warn("Image reference release failed after delete", { ownerId, imageRef: path, error: String(err) }); + } + }); + return { ...actual, decrementImageRef, releaseImageRef }; +}); + +import { registerGovernanceFunction } from "../src/functions/governance.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerEvictFunction } from "../src/functions/evict.js"; +import { registerAutoForgetFunction } from "../src/functions/auto-forget.js"; +import { registerRetentionFunctions } from "../src/functions/retention.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { KV } from "../src/state/schema.js"; +import { getSearchIndex } from "../src/functions/search.js"; +import { memoryToObservation } from "../src/state/memory-utils.js"; +import type { Memory, RetentionScore } from "../src/types.js"; + +type Gate = { scope: string; key: string; reached: () => void; release: Promise }; + +function mockKV() { + const store = new Map>(); + let gate: Gate | null = null; + return { + store, + gateNextSet(scope: string, key: string) { + let reached!: () => void; + let release!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + gate = { scope, key, reached, release: releasePromise }; + return { reached: reachedPromise, release }; + }, + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (gate && gate.scope === scope && gate.key === key) { + const current = gate; + gate = null; + current.reached(); + await current.release; + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) return { success: true }; + return fn(payload); + }, + }; +} + +function makeMemory(id: string, overrides: Partial = {}): Memory { + return { + id, + createdAt: "2026-02-01T00:00:00Z", + updatedAt: "2026-02-01T00:00:00Z", + type: "fact", + title: `Memory ${id}`, + content: `Content for ${id}`, + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: true, + ...overrides, + }; +} + +async function holdRememberLockAndRewrite(kv: ReturnType, id: string) { + let reached!: () => void; + let release!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const done = withKeyedLock("mem:remember", async () => { + const current = await kv.get(KV.memories, id); + reached(); + await releasePromise; + if (current) await kv.set(KV.memories, id, { ...current, isLatest: false }); + }); + await reachedPromise; + return { release, done }; +} + +describe("memory delete paths hold the remember lock", () => { + it("governance delete waits for an in-flight evolve and the memory stays deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerGovernanceFunction(sdk as never, kv as never); + registerRelationsFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_x", makeMemory("mem_x")); + + const gate = kv.gateNextSet(KV.memories, "mem_x"); + const evolve = sdk.trigger("mem::evolve", { memoryId: "mem_x", newContent: "updated content" }); + await gate.reached; + + const deletion = sdk.trigger("mem::governance-delete", { memoryIds: ["mem_x"] }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_x")).not.toBeNull(); + + gate.release(); + await evolve; + const result = (await deletion) as { deleted: number }; + + expect(result.deleted).toBe(1); + expect(await kv.get(KV.memories, "mem_x")).toBeNull(); + }); + + it("evict TTL delete waits for the lock holder and does not get undone", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ttl", makeMemory("mem_ttl", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const holder = await holdRememberLockAndRewrite(kv, "mem_ttl"); + const evict = sdk.trigger("mem::evict", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_ttl")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await evict) as { expiredMemories: number }; + + expect(result.expiredMemories).toBe(1); + expect(await kv.get(KV.memories, "mem_ttl")).toBeNull(); + }); + + it("evict skips a memory that another writer already deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_gone", makeMemory("mem_gone", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.memories, "mem_gone"); + }); + const result = (await sdk.trigger("mem::evict", {})) as { expiredMemories: number }; + await done; + + expect(result.expiredMemories).toBe(0); + }); + + it("auto-forget TTL delete waits for the lock holder and does not get undone", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerAutoForgetFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_af", makeMemory("mem_af", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const holder = await holdRememberLockAndRewrite(kv, "mem_af"); + const run = sdk.trigger("mem::auto-forget", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_af")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await run) as { ttlExpired: string[] }; + + expect(result.ttlExpired).toEqual(["mem_af"]); + expect(await kv.get(KV.memories, "mem_af")).toBeNull(); + }); + + it("auto-forget releases the image of the record it deleted, outside the remember lock", async () => { + imageReleases.length = 0; + const kv = mockKV(); + const sdk = mockSdk(); + registerAutoForgetFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_img", makeMemory("mem_img", { forgetAfter: "2020-01-01T00:00:00Z", imageRef: "old.png" })); + + let release!: () => void; + let reached!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const holder = withKeyedLock("mem:remember", async () => { + reached(); + await releasePromise; + const current = await kv.get(KV.memories, "mem_img"); + await kv.set(KV.memories, "mem_img", { ...current!, imageRef: "new.png" }); + }); + await reachedPromise; + const run = sdk.trigger("mem::auto-forget", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + release(); + await holder; + await run; + + expect(await kv.get(KV.memories, "mem_img")).toBeNull(); + expect(imageReleases).toEqual([{ path: "new.png", lockHeld: false }]); + }); + + it("evict releases the image of the record it deleted, not the pre-lock snapshot", async () => { + imageReleases.length = 0; + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ev", makeMemory("mem_ev", { forgetAfter: "2020-01-01T00:00:00Z", imageRef: "old.png" })); + + let release!: () => void; + let reached!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const holder = withKeyedLock("mem:remember", async () => { + reached(); + await releasePromise; + const current = await kv.get(KV.memories, "mem_ev"); + await kv.set(KV.memories, "mem_ev", { ...current!, imageRef: "new.png" }); + }); + await reachedPromise; + const run = sdk.trigger("mem::evict", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + release(); + await holder; + await run; + + expect(await kv.get(KV.memories, "mem_ev")).toBeNull(); + expect(imageReleases).toEqual([{ path: "new.png", lockHeld: false }]); + }); +}); + +function score(memoryId: string, value: number): RetentionScore { + return { + memoryId, + source: "episodic", + score: value, + salience: value, + temporalDecay: 1, + reinforcementBoost: 0, + lastAccessed: "2026-01-01T00:00:00Z", + accessCount: 0, + }; +} + +describe("retention eviction holds the remember lock", () => { + it("waits for the lock holder and the memory stays deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ret", makeMemory("mem_ret")); + await kv.set(KV.retentionScores, "mem_ret", score("mem_ret", 0.01)); + + const holder = await holdRememberLockAndRewrite(kv, "mem_ret"); + const evict = sdk.trigger("mem::retention-evict", { threshold: 0.5 }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_ret")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await evict) as { evicted: number }; + + expect(result.evicted).toBe(1); + expect(await kv.get(KV.memories, "mem_ret")).toBeNull(); + }); + + it("drops the score of a memory deleted while eviction waited so it cannot hold a slot", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.memories, "mem_a", makeMemory("mem_a")); + await kv.set(KV.memories, "mem_b", makeMemory("mem_b")); + await kv.set(KV.retentionScores, "mem_a", score("mem_a", 0.01)); + await kv.set(KV.retentionScores, "mem_b", score("mem_b", 0.02)); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.memories, "mem_a"); + }); + const first = (await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 })) as { evicted: number }; + await done; + + expect(first.evicted).toBe(0); + expect(await kv.get(KV.retentionScores, "mem_a")).toBeNull(); + + const second = (await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 })) as { evicted: number }; + expect(second.evicted).toBe(1); + expect(await kv.get(KV.memories, "mem_b")).toBeNull(); + }); + + it("keeps the score when the memory still lives in the other scope", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.semantic, "mem_s", makeMemory("mem_s")); + await kv.set(KV.retentionScores, "mem_s", { ...score("mem_s", 0.01), source: undefined }); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.semantic, "mem_s"); + await kv.set(KV.memories, "mem_s", makeMemory("mem_s")); + }); + await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 }); + await done; + + expect(await kv.get(KV.retentionScores, "mem_s")).not.toBeNull(); + }); + + it("finishes cleanup and continues the sweep when an image release fails", async () => { + imageFailures.clear(); + imageFailures.add("broken.png"); + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + const broken = makeMemory("mem_img_a", { imageRef: "broken.png" }); + const next = makeMemory("mem_img_b"); + for (const mem of [broken, next]) { + await kv.set(KV.memories, mem.id, mem); + getSearchIndex().add(memoryToObservation(mem), "memory"); + } + await kv.set(KV.retentionScores, "mem_img_a", score("mem_img_a", 0.01)); + await kv.set(KV.retentionScores, "mem_img_b", score("mem_img_b", 0.02)); + + const result = (await sdk.trigger("mem::retention-evict", { threshold: 0.5 })) as { evicted: number }; + imageFailures.clear(); + + expect(result.evicted).toBe(2); + for (const id of ["mem_img_a", "mem_img_b"]) { + expect(await kv.get(KV.memories, id)).toBeNull(); + expect(await kv.get(KV.retentionScores, id)).toBeNull(); + expect(getSearchIndex().has(id)).toBe(false); + } + }); + + it("records an audit entry when it removes orphan scores and evicts nothing", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.retentionScores, "mem_orphan", score("mem_orphan", 0.01)); + + const result = (await sdk.trigger("mem::retention-evict", { threshold: 0.5 })) as { evicted: number }; + + expect(result.evicted).toBe(0); + expect(await kv.get(KV.retentionScores, "mem_orphan")).toBeNull(); + const audits = [...kv.store.entries()] + .filter(([scope]) => scope.startsWith("mem:audit")) + .flatMap(([, rows]) => [...rows.values()]) as Array<{ functionId: string; targetIds: string[]; details?: Record }>; + expect(audits).toContainEqual(expect.objectContaining({ + functionId: "mem::retention-evict", + targetIds: ["mem_orphan"], + details: expect.objectContaining({ resource: "retention_score", removed: 1 }), + })); + }); +});