From cf9a7381917ea4db7bd574f85c397354a98f1e5f Mon Sep 17 00:00:00 2001 From: Rohit Ghumare <48523873+rohitg00@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:28:08 +0530 Subject: [PATCH] fix(memory): hold the remember lock on every memory delete path (#1746) * fix(memory): hold the remember lock on every memory delete path Governance delete, evict (TTL and old non-latest), auto-forget TTL and retention eviction deleted memory records without the mem:remember lock that remember, evolve and forget hold. A delete that landed between an evolve's read and its write was undone when evolve wrote the record back. Each path now re-reads the record under the lock and deletes it there, skipping records another writer already removed. The lock is taken per memory, so a long sweep does not block remember. withReentrantKeyedLock runs the body directly when the caller already holds the key. Fixes #1728 * fix(memory): release images outside the remember lock and drop orphan retention scores deleteMemoryUnderLock now returns the record it read and deleted under mem:remember. Auto-forget and retention release that record's image after the lock is released, so image I/O no longer serializes every memory write and a record replaced while the sweep waited releases its own image instead of the stale one. Retention eviction deletes the score of a candidate that is gone from both memory scopes, so a memory forgotten while the sweep waited no longer holds a maxEvict slot on every later run. A score whose memory still lives in the other scope is kept. Adds retention eviction race, orphan score and image release tests. * fix(retention): finish delete cleanup when an image release fails and audit orphan scores A rejected decrementImageRef after the locked delete skipped the score, access log and search cleanup in retention eviction and aborted the auto-forget TTL sweep. Release images through releaseImageRef, which logs the failure with the owning record id and lets the rest of the cleanup run. Removing retention scores for memories that no longer exist now records a mem::retention-evict audit entry even when nothing was evicted. * fix(evict): release the image of the record deleted under the lock mem::evict read imageRef from the snapshot taken before it waited for mem:remember. If another writer changed imageRef in that window, evict released the old reference and leaked the new one. Use the record that deleteMemoryUnderLock returns, and route every image release in evict through releaseImageRef so a failed release no longer cuts cleanup short. * test(memory): cover the real releaseImageRef failure path The delete-lock suite mocks releaseImageRef, so a regression in its catch path would leave the suite green. A separate test now runs the real helper against a store whose write fails and checks that it logs the failure and does not throw, plus the normal decrement. Signed-off-by: Rohit Ghumare --------- Signed-off-by: Rohit Ghumare --- src/functions/auto-forget.ts | 11 +- src/functions/evict.ts | 27 +-- src/functions/governance.ts | 5 +- src/functions/image-refs.ts | 13 ++ src/functions/retention.ts | 24 +- src/state/keyed-mutex.ts | 4 + src/state/memory-utils.ts | 15 ++ test/image-ref-release.test.ts | 50 ++++ test/memory-delete-lock.test.ts | 388 ++++++++++++++++++++++++++++++++ 9 files changed, 511 insertions(+), 26 deletions(-) create mode 100644 test/image-ref-release.test.ts create mode 100644 test/memory-delete-lock.test.ts diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts index 3e7543abaf..42ee095fbd 100644 --- a/src/functions/auto-forget.ts +++ b/src/functions/auto-forget.ts @@ -3,7 +3,7 @@ import { markCaptureEventDeleted } from "../capture/event-record.js"; import type { Memory, CompressedObservation, Session } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV, collectAll } from "../state/kv.js"; -import { supersededAt } from "../state/memory-utils.js"; +import { deleteMemoryUnderLock, supersededAt } from "../state/memory-utils.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { unindexObservationSession } from "../state/obs-index.js"; import { recordAudit } from "./audit.js"; @@ -31,7 +31,7 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void { async (data: { dryRun?: boolean }): Promise => { const dryRun = data?.dryRun ?? false; const now = Date.now(); - const { decrementImageRef } = await import("./image-refs.js"); + const { decrementImageRef, releaseImageRef } = await import("./image-refs.js"); const result: AutoForgetResult = { ttlExpired: [], @@ -51,10 +51,9 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void { result.ttlExpired.push(mem.id); deletedIds.add(mem.id); if (!dryRun) { - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } - await kv.delete(KV.memories, mem.id); + const removed = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!removed) continue; + if (removed.imageRef) await releaseImageRef(kv, sdk, mem.id, removed.imageRef); facetIds.add(mem.id); await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], { resource: "memory", diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 33600d3cb0..7c8d0d5c62 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -10,6 +10,7 @@ import type { import { KV } from "../state/schema.js"; import { StateKV, collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import { removeSessionFromProjectIndex } from "../state/session-index.js"; import { unindexObservationSession } from "../state/obs-index.js"; import { isConsolidationEnabled } from "../config.js"; @@ -115,7 +116,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction("mem::evict", async (data: { dryRun?: boolean }): Promise => { const dryRun = data?.dryRun ?? false; - const { decrementImageRef } = await import("./image-refs.js"); + const { releaseImageRef } = await import("./image-refs.js"); let indexCleaned = false; const removeObservation = ( sessionId: string, @@ -271,8 +272,8 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { continue; } await unindexObservationSession(kv, o.id).catch(() => {}); - if (removed.imageData) await decrementImageRef(kv, sdk, removed.imageData); - if (removed.imageRef && removed.imageRef !== removed.imageData) await decrementImageRef(kv, sdk, removed.imageRef); + if (removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageData); + if (removed.imageRef && removed.imageRef !== removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageRef); await recordAudit(kv, "delete", "mem::evict", [o.id], { resource: "observation", reason: "low_importance_old_observation", @@ -319,8 +320,8 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { continue; } await unindexObservationSession(kv, o.id).catch(() => {}); - if (removed.imageData) await decrementImageRef(kv, sdk, removed.imageData); - if (removed.imageRef && removed.imageRef !== removed.imageData) await decrementImageRef(kv, sdk, removed.imageRef); + if (removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageData); + if (removed.imageRef && removed.imageRef !== removed.imageData) await releaseImageRef(kv, sdk, o.id, removed.imageRef); await recordAudit(kv, "delete", "mem::evict", [o.id], { resource: "observation", reason: "project_observation_cap", @@ -344,8 +345,10 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { stats.expiredMemories++; evictedMemIds.add(mem.id); } else { + let deleted: Memory | null; try { - await kv.delete(KV.memories, mem.id); + deleted = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!deleted) continue; deletedMemIds.add(mem.id); stats.expiredMemories++; evictedMemIds.add(mem.id); @@ -361,9 +364,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { getSearchIndex().remove(mem.id); vectorIndexRemove(mem.id); indexCleaned = true; - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } + if (deleted.imageRef) await releaseImageRef(kv, sdk, mem.id, deleted.imageRef); await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "expired_memory", @@ -384,8 +385,10 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { if (dryRun) { stats.nonLatestMemories++; } else { + let deleted: Memory | null; try { - await kv.delete(KV.memories, mem.id); + deleted = await deleteMemoryUnderLock(kv, KV.memories, mem.id); + if (!deleted) continue; deletedMemIds.add(mem.id); stats.nonLatestMemories++; } catch (err) { @@ -400,9 +403,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { getSearchIndex().remove(mem.id); vectorIndexRemove(mem.id); indexCleaned = true; - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } + if (deleted.imageRef) await releaseImageRef(kv, sdk, mem.id, deleted.imageRef); await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "old_non_latest_memory", diff --git a/src/functions/governance.ts b/src/functions/governance.ts index 41ac200c62..e6b123f536 100644 --- a/src/functions/governance.ts +++ b/src/functions/governance.ts @@ -3,6 +3,7 @@ import type { Memory, GovernanceFilter, AuditEntry } from "../types.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { collectAll } from "../state/kv.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import { recordAudit, safeAudit, queryAudit } from "./audit.js"; import { deleteAccessLog } from "./access-tracker.js"; import { deleteFacetsForTargets } from "./facets.js"; @@ -21,9 +22,7 @@ async function deleteMemoryRecord( kv: StateKV, id: string, ): Promise<"deleted" | "missing"> { - const mem = await kv.get(KV.memories, id); - if (!mem) return "missing"; - await kv.delete(KV.memories, id); + if (!(await deleteMemoryUnderLock(kv, KV.memories, id))) return "missing"; await deleteAccessLog(kv, id); getSearchIndex().remove(id); vectorIndexRemove(id); diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.ts index 918a7e7c46..75012f0448 100644 --- a/src/functions/image-refs.ts +++ b/src/functions/image-refs.ts @@ -3,6 +3,7 @@ import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { deleteImage, touchImage } from "../utils/image-store.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { logger } from "../logger.js"; export async function getImageRefCount(kv: StateKV, filePath: string): Promise { const count = await kv.get(KV.imageRefs, filePath); @@ -36,3 +37,15 @@ export async function decrementImageRef(kv: StateKV, sdk: IIIClient, filePath: s } }); } + +export async function releaseImageRef(kv: StateKV, sdk: IIIClient, ownerId: string, filePath: string): Promise { + try { + await decrementImageRef(kv, sdk, filePath); + } catch (err) { + logger.warn("Image reference release failed after delete", { + ownerId, + imageRef: filePath, + error: err instanceof Error ? err.message : String(err), + }); + } +} diff --git a/src/functions/retention.ts b/src/functions/retention.ts index 97c05db910..a40e5f4a78 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -8,6 +8,7 @@ import type { import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { collectAll } from "../state/kv.js"; +import { deleteMemoryUnderLock } from "../state/memory-utils.js"; import type { AccessLog } from "./access-tracker.js"; import { emptyAccessLog, @@ -306,7 +307,7 @@ export function registerRetentionFunctions( ? data.maxEvict : 50; const maxEvict = Math.min(1000, Math.max(0, maxEvictRaw)); - const { decrementImageRef } = await import("./image-refs.js"); + const { releaseImageRef } = await import("./image-refs.js"); const allScores = await kv.list(KV.retentionScores); const candidates = allScores @@ -373,10 +374,12 @@ export function registerRetentionFunctions( continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + const removed = await deleteMemoryUnderLock(kv, scope, candidate.memoryId); + if (!removed) { + missingIds.push(candidate.memoryId); + continue; } - await kv.delete(scope, candidate.memoryId); + if (removed.imageRef) await releaseImageRef(kv, sdk, candidate.memoryId, removed.imageRef); evictedIds.push(candidate.memoryId); evicted++; if (resolvedSource === "semantic") evictedSemantic++; @@ -390,6 +393,12 @@ export function registerRetentionFunctions( } } + const orphanScoreIds: string[] = []; + for (const id of missingIds) { + if (await kv.get(KV.memories, id) || await kv.get(KV.semantic, id)) continue; + await kv.delete(KV.retentionScores, id); + orphanScoreIds.push(id); + } await deleteFacetsForTargets(kv, [...evictedIds, ...missingIds]); // Retention eviction is a structural delete path that removes @@ -408,6 +417,13 @@ export function registerRetentionFunctions( reason: "retention score below threshold", }); } + if (orphanScoreIds.length > 0) { + await recordAudit(kv, "delete", "mem::retention-evict", orphanScoreIds, { + resource: "retention_score", + removed: orphanScoreIds.length, + reason: "retention score for a memory that no longer exists", + }); + } logger.info("Retention-based eviction complete", { evicted, diff --git a/src/state/keyed-mutex.ts b/src/state/keyed-mutex.ts index 6ac2ac6ad7..308a5e7282 100644 --- a/src/state/keyed-mutex.ts +++ b/src/state/keyed-mutex.ts @@ -58,6 +58,10 @@ export function withKeyedLock( ); } +export function withReentrantKeyedLock(key: string, fn: () => Promise): Promise { + return isKeyedLockHeld(key) ? fn() : withKeyedLock(key, fn); +} + export function withKeyedLocks(keys: string[], fn: () => Promise): Promise { const ordered = [...new Set(keys)].sort(); const acquire = (index: number): Promise => index === ordered.length diff --git a/src/state/memory-utils.ts b/src/state/memory-utils.ts index 8c7cbe329e..86a463efa2 100644 --- a/src/state/memory-utils.ts +++ b/src/state/memory-utils.ts @@ -1,7 +1,22 @@ import type { CompressedObservation, Lesson, Memory } from "../types.js"; +import type { StateKV } from "./kv.js"; +import { withReentrantKeyedLock } from "./keyed-mutex.js"; const MAX_DATE_MS = 8.64e15; +export function deleteMemoryUnderLock( + kv: StateKV, + scope: string, + id: string, +): Promise { + return withReentrantKeyedLock("mem:remember", async () => { + const current = await kv.get(scope, id); + if (!current) return null; + await kv.delete(scope, id); + return current; + }); +} + export function supersededAt(previous: string | undefined, now: string): string { const prior = previous ? Date.parse(previous) : Number.NaN; const current = Date.parse(now); diff --git a/test/image-ref-release.test.ts b/test/image-ref-release.test.ts new file mode 100644 index 0000000000..7f9784231b --- /dev/null +++ b/test/image-ref-release.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { warn } = vi.hoisted(() => ({ warn: vi.fn() })); +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn, error: vi.fn() }, +})); + +import { releaseImageRef } from "../src/functions/image-refs.js"; +import { KV } from "../src/state/schema.js"; + +function kvWithFailingWrite(count: number) { + return { + get: async (scope: string) => (scope === KV.imageRefs ? count : null), + set: async () => { + throw new Error("image ref write failed"); + }, + delete: async () => {}, + list: async () => [], + }; +} + +describe("releaseImageRef", () => { + beforeEach(() => warn.mockClear()); + + it("logs a failed image ref write and does not throw", async () => { + const sdk = { trigger: vi.fn() }; + await expect(releaseImageRef(kvWithFailingWrite(2) as never, sdk as never, "mem_1", "/tmp/img-a.png")).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith("Image reference release failed after delete", { + ownerId: "mem_1", + imageRef: "/tmp/img-a.png", + error: "image ref write failed", + }); + }); + + it("decrements the count when the write succeeds", async () => { + const writes: Array<[string, string, unknown]> = []; + const kv = { + get: async () => 3, + set: async (scope: string, key: string, value: unknown) => { + writes.push([scope, key, value]); + return value; + }, + delete: async () => {}, + list: async () => [], + }; + await releaseImageRef(kv as never, { trigger: vi.fn() } as never, "mem_2", "/tmp/img-b.png"); + expect(writes).toEqual([[KV.imageRefs, "/tmp/img-b.png", 2]]); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/test/memory-delete-lock.test.ts b/test/memory-delete-lock.test.ts new file mode 100644 index 0000000000..66fd6d6cdc --- /dev/null +++ b/test/memory-delete-lock.test.ts @@ -0,0 +1,388 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/config.js", async (importOriginal) => ({ + ...(await importOriginal()), + isConsolidationEnabled: () => false, +})); + +const imageReleases: Array<{ path: string; lockHeld: boolean }> = []; +const imageFailures = new Set(); +vi.mock("../src/functions/image-refs.js", async (importOriginal) => { + const actual = await importOriginal(); + const decrementImageRef = vi.fn(async (_kv: unknown, _sdk: unknown, path: string) => { + const { isKeyedLockHeld } = await import("../src/state/keyed-mutex.js"); + imageReleases.push({ path, lockHeld: isKeyedLockHeld("mem:remember") }); + if (imageFailures.has(path)) throw new Error("image ref write failed"); + }); + const releaseImageRef = vi.fn(async (kv: unknown, sdk: unknown, ownerId: string, path: string) => { + try { + await decrementImageRef(kv, sdk, path); + } catch (err) { + const { logger } = await import("../src/logger.js"); + logger.warn("Image reference release failed after delete", { ownerId, imageRef: path, error: String(err) }); + } + }); + return { ...actual, decrementImageRef, releaseImageRef }; +}); + +import { registerGovernanceFunction } from "../src/functions/governance.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerEvictFunction } from "../src/functions/evict.js"; +import { registerAutoForgetFunction } from "../src/functions/auto-forget.js"; +import { registerRetentionFunctions } from "../src/functions/retention.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { KV } from "../src/state/schema.js"; +import { getSearchIndex } from "../src/functions/search.js"; +import { memoryToObservation } from "../src/state/memory-utils.js"; +import type { Memory, RetentionScore } from "../src/types.js"; + +type Gate = { scope: string; key: string; reached: () => void; release: Promise }; + +function mockKV() { + const store = new Map>(); + let gate: Gate | null = null; + return { + store, + gateNextSet(scope: string, key: string) { + let reached!: () => void; + let release!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + gate = { scope, key, reached, release: releasePromise }; + return { reached: reachedPromise, release }; + }, + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (gate && gate.scope === scope && gate.key === key) { + const current = gate; + gate = null; + current.reached(); + await current.release; + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) return { success: true }; + return fn(payload); + }, + }; +} + +function makeMemory(id: string, overrides: Partial = {}): Memory { + return { + id, + createdAt: "2026-02-01T00:00:00Z", + updatedAt: "2026-02-01T00:00:00Z", + type: "fact", + title: `Memory ${id}`, + content: `Content for ${id}`, + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: true, + ...overrides, + }; +} + +async function holdRememberLockAndRewrite(kv: ReturnType, id: string) { + let reached!: () => void; + let release!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const done = withKeyedLock("mem:remember", async () => { + const current = await kv.get(KV.memories, id); + reached(); + await releasePromise; + if (current) await kv.set(KV.memories, id, { ...current, isLatest: false }); + }); + await reachedPromise; + return { release, done }; +} + +describe("memory delete paths hold the remember lock", () => { + it("governance delete waits for an in-flight evolve and the memory stays deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerGovernanceFunction(sdk as never, kv as never); + registerRelationsFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_x", makeMemory("mem_x")); + + const gate = kv.gateNextSet(KV.memories, "mem_x"); + const evolve = sdk.trigger("mem::evolve", { memoryId: "mem_x", newContent: "updated content" }); + await gate.reached; + + const deletion = sdk.trigger("mem::governance-delete", { memoryIds: ["mem_x"] }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_x")).not.toBeNull(); + + gate.release(); + await evolve; + const result = (await deletion) as { deleted: number }; + + expect(result.deleted).toBe(1); + expect(await kv.get(KV.memories, "mem_x")).toBeNull(); + }); + + it("evict TTL delete waits for the lock holder and does not get undone", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ttl", makeMemory("mem_ttl", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const holder = await holdRememberLockAndRewrite(kv, "mem_ttl"); + const evict = sdk.trigger("mem::evict", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_ttl")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await evict) as { expiredMemories: number }; + + expect(result.expiredMemories).toBe(1); + expect(await kv.get(KV.memories, "mem_ttl")).toBeNull(); + }); + + it("evict skips a memory that another writer already deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_gone", makeMemory("mem_gone", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.memories, "mem_gone"); + }); + const result = (await sdk.trigger("mem::evict", {})) as { expiredMemories: number }; + await done; + + expect(result.expiredMemories).toBe(0); + }); + + it("auto-forget TTL delete waits for the lock holder and does not get undone", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerAutoForgetFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_af", makeMemory("mem_af", { forgetAfter: "2020-01-01T00:00:00Z" })); + + const holder = await holdRememberLockAndRewrite(kv, "mem_af"); + const run = sdk.trigger("mem::auto-forget", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_af")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await run) as { ttlExpired: string[] }; + + expect(result.ttlExpired).toEqual(["mem_af"]); + expect(await kv.get(KV.memories, "mem_af")).toBeNull(); + }); + + it("auto-forget releases the image of the record it deleted, outside the remember lock", async () => { + imageReleases.length = 0; + const kv = mockKV(); + const sdk = mockSdk(); + registerAutoForgetFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_img", makeMemory("mem_img", { forgetAfter: "2020-01-01T00:00:00Z", imageRef: "old.png" })); + + let release!: () => void; + let reached!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const holder = withKeyedLock("mem:remember", async () => { + reached(); + await releasePromise; + const current = await kv.get(KV.memories, "mem_img"); + await kv.set(KV.memories, "mem_img", { ...current!, imageRef: "new.png" }); + }); + await reachedPromise; + const run = sdk.trigger("mem::auto-forget", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + release(); + await holder; + await run; + + expect(await kv.get(KV.memories, "mem_img")).toBeNull(); + expect(imageReleases).toEqual([{ path: "new.png", lockHeld: false }]); + }); + + it("evict releases the image of the record it deleted, not the pre-lock snapshot", async () => { + imageReleases.length = 0; + const kv = mockKV(); + const sdk = mockSdk(); + registerEvictFunction(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ev", makeMemory("mem_ev", { forgetAfter: "2020-01-01T00:00:00Z", imageRef: "old.png" })); + + let release!: () => void; + let reached!: () => void; + const reachedPromise = new Promise((resolve) => { reached = resolve; }); + const releasePromise = new Promise((resolve) => { release = resolve; }); + const holder = withKeyedLock("mem:remember", async () => { + reached(); + await releasePromise; + const current = await kv.get(KV.memories, "mem_ev"); + await kv.set(KV.memories, "mem_ev", { ...current!, imageRef: "new.png" }); + }); + await reachedPromise; + const run = sdk.trigger("mem::evict", {}); + await new Promise((resolve) => setTimeout(resolve, 10)); + release(); + await holder; + await run; + + expect(await kv.get(KV.memories, "mem_ev")).toBeNull(); + expect(imageReleases).toEqual([{ path: "new.png", lockHeld: false }]); + }); +}); + +function score(memoryId: string, value: number): RetentionScore { + return { + memoryId, + source: "episodic", + score: value, + salience: value, + temporalDecay: 1, + reinforcementBoost: 0, + lastAccessed: "2026-01-01T00:00:00Z", + accessCount: 0, + }; +} + +describe("retention eviction holds the remember lock", () => { + it("waits for the lock holder and the memory stays deleted", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.memories, "mem_ret", makeMemory("mem_ret")); + await kv.set(KV.retentionScores, "mem_ret", score("mem_ret", 0.01)); + + const holder = await holdRememberLockAndRewrite(kv, "mem_ret"); + const evict = sdk.trigger("mem::retention-evict", { threshold: 0.5 }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await kv.get(KV.memories, "mem_ret")).not.toBeNull(); + + holder.release(); + await holder.done; + const result = (await evict) as { evicted: number }; + + expect(result.evicted).toBe(1); + expect(await kv.get(KV.memories, "mem_ret")).toBeNull(); + }); + + it("drops the score of a memory deleted while eviction waited so it cannot hold a slot", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.memories, "mem_a", makeMemory("mem_a")); + await kv.set(KV.memories, "mem_b", makeMemory("mem_b")); + await kv.set(KV.retentionScores, "mem_a", score("mem_a", 0.01)); + await kv.set(KV.retentionScores, "mem_b", score("mem_b", 0.02)); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.memories, "mem_a"); + }); + const first = (await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 })) as { evicted: number }; + await done; + + expect(first.evicted).toBe(0); + expect(await kv.get(KV.retentionScores, "mem_a")).toBeNull(); + + const second = (await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 })) as { evicted: number }; + expect(second.evicted).toBe(1); + expect(await kv.get(KV.memories, "mem_b")).toBeNull(); + }); + + it("keeps the score when the memory still lives in the other scope", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.semantic, "mem_s", makeMemory("mem_s")); + await kv.set(KV.retentionScores, "mem_s", { ...score("mem_s", 0.01), source: undefined }); + + const done = withKeyedLock("mem:remember", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await kv.delete(KV.semantic, "mem_s"); + await kv.set(KV.memories, "mem_s", makeMemory("mem_s")); + }); + await sdk.trigger("mem::retention-evict", { threshold: 0.5, maxEvict: 1 }); + await done; + + expect(await kv.get(KV.retentionScores, "mem_s")).not.toBeNull(); + }); + + it("finishes cleanup and continues the sweep when an image release fails", async () => { + imageFailures.clear(); + imageFailures.add("broken.png"); + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + const broken = makeMemory("mem_img_a", { imageRef: "broken.png" }); + const next = makeMemory("mem_img_b"); + for (const mem of [broken, next]) { + await kv.set(KV.memories, mem.id, mem); + getSearchIndex().add(memoryToObservation(mem), "memory"); + } + await kv.set(KV.retentionScores, "mem_img_a", score("mem_img_a", 0.01)); + await kv.set(KV.retentionScores, "mem_img_b", score("mem_img_b", 0.02)); + + const result = (await sdk.trigger("mem::retention-evict", { threshold: 0.5 })) as { evicted: number }; + imageFailures.clear(); + + expect(result.evicted).toBe(2); + for (const id of ["mem_img_a", "mem_img_b"]) { + expect(await kv.get(KV.memories, id)).toBeNull(); + expect(await kv.get(KV.retentionScores, id)).toBeNull(); + expect(getSearchIndex().has(id)).toBe(false); + } + }); + + it("records an audit entry when it removes orphan scores and evicts nothing", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerRetentionFunctions(sdk as never, kv as never); + await kv.set(KV.retentionScores, "mem_orphan", score("mem_orphan", 0.01)); + + const result = (await sdk.trigger("mem::retention-evict", { threshold: 0.5 })) as { evicted: number }; + + expect(result.evicted).toBe(0); + expect(await kv.get(KV.retentionScores, "mem_orphan")).toBeNull(); + const audits = [...kv.store.entries()] + .filter(([scope]) => scope.startsWith("mem:audit")) + .flatMap(([, rows]) => [...rows.values()]) as Array<{ functionId: string; targetIds: string[]; details?: Record }>; + expect(audits).toContainEqual(expect.objectContaining({ + functionId: "mem::retention-evict", + targetIds: ["mem_orphan"], + details: expect.objectContaining({ resource: "retention_score", removed: 1 }), + })); + }); +});