From 689204139df4cf89ea3269b981f55abf5b5f74b1 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:49:43 -0500 Subject: [PATCH] fix: serialize Memory writes and provider recovery probes Recheck consolidation predecessors after provider calls and keep one\nprobe in flight until it settles. Preserve eviction bookkeeping after\na successful row deletion, even if image release fails. Refs: #474 Generated-By: gpt-6.2-sol --- CLAUDE.md | 12 +- src/functions/auto-forget.ts | 29 +++-- src/functions/consolidate.ts | 119 ++++++++++--------- src/functions/evict.ts | 39 ++++-- src/functions/governance.ts | 34 +++--- src/functions/relations.ts | 115 +++++++++--------- src/functions/remember.ts | 22 ++-- src/functions/retention.ts | 21 ++-- src/providers/circuit-breaker.ts | 36 ++++-- src/providers/resilient.ts | 24 ++-- src/state/memory-utils.ts | 2 + test/circuit-breaker.test.ts | 74 ++++++++++++ test/consolidate-dedup.test.ts | 55 +++++++++ test/evict.test.ts | 21 +++- test/memory-write-locks.test.ts | 197 +++++++++++++++++++++++++++++++ test/resilient-half-open.test.ts | 74 ++++++++++++ 16 files changed, 689 insertions(+), 185 deletions(-) create mode 100644 test/memory-write-locks.test.ts create mode 100644 test/resilient-half-open.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index e8de79249d..ee96526d00 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -237,6 +237,16 @@ Auto-forget and retention release a Memory's image reference after its indexed d A failed release is logged without blocking deletion bookkeeping; it may leave an unused image. Concurrent deletion locking is outside this contract. +### Memory writes and provider recovery +`MEMORY_WRITE_LOCK` (`src/state/memory-utils.ts`) serializes Memory saves, deletes, +relations, evolution, and auto-forget demotion. Consolidation checks the predecessor again +under the lock after the provider call. Release the lock before graph eviction. Import and +snapshot restore keep their own replacement semantics. Eviction decrements image references +after deleting the row and logs a failed release without skipping bookkeeping. +`ResilientProvider` admits one half-open probe per operation after acquiring a provider slot. +The reservation lasts until the call settles; busy and content-filter responses release it, +and probe generations discard stale completions. An open circuit rejects before waiting for a slot. + ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap eviction, `mem::auto-forget`, `mem::forget`, `mem::retention-evict`, `mem::governance-delete`, @@ -463,7 +473,7 @@ short forms count only beside an ordinal, a year, or after "on". The in-process BM25 from content at boot, so there is no migration. ### Search and consolidation -`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; cross-writer snapshot races are not covered by this lock. +`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; the write lock does not make both writes transactional. ### Search on the event loop Search runs on the daemon's one event loop, which also answers `/livez` and the hooks. BM25 diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts index ccf45da817..c14743a359 100644 --- a/src/functions/auto-forget.ts +++ b/src/functions/auto-forget.ts @@ -4,9 +4,10 @@ import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { recordAudit } from "./audit.js"; import { deleteIndexed, deindexSupersededMemory } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { lowerObservationCounts } from "./observe.js"; import { evictGraphForSources } from "./graph-eviction.js"; -import { refersToDifferentDates } from "../state/memory-utils.js"; +import { refersToDifferentDates, MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { logger } from "../logger.js"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -53,12 +54,16 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void { result.ttlExpired.push(mem.id); deletedIds.add(mem.id); if (!dryRun) { - await deleteIndexed(kv, KV.memories, mem.id); - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { - logger.warn("Auto-forget could not release image reference", { memoryId: mem.id, error: String(error) }); - }); - } + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((error) => { + logger.warn("Auto-forget could not release image reference", { memoryId: current.id, error: String(error) }); + }); + } + }); evictedSourceIds.push(mem.id); await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], { resource: "memory", @@ -149,9 +154,13 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void { new Date(memB.createdAt).getTime() ? memA : memB; - older.isLatest = false; - await kv.set(KV.memories, older.id, older); - deindexSupersededMemory(older.id); + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, older.id); + if (!current) return; + current.isLatest = false; + await kv.set(KV.memories, current.id, current); + deindexSupersededMemory(current.id); + }); await recordAudit(kv, "forget", "mem::auto-forget", [older.id], { resource: "memory", reason: "auto-forget contradiction", diff --git a/src/functions/consolidate.ts b/src/functions/consolidate.ts index f2e431d140..debc0ff0e7 100644 --- a/src/functions/consolidate.ts +++ b/src/functions/consolidate.ts @@ -7,6 +7,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; import { deindexSupersededMemory, indexNewMemoryVersion } from "./search.js"; @@ -113,7 +114,7 @@ export function registerConsolidateFunction( } let consolidated = 0; - const existingMemories = await kv.list(KV.memories); + const existingMemories = structuredClone(await kv.list(KV.memories)); const writtenTitles = new Set(); const scopedProject = typeof data.project === "string" && data.project.trim().length > 0 @@ -185,62 +186,66 @@ export function registerConsolidateFunction( m.title.toLowerCase() === parsed.title.toLowerCase(), ); - if (existingMatch) { - const evolved: Memory = { - id: generateId("mem"), - createdAt: now, - updatedAt: now, - ...parsed, - version: (existingMatch.version || 1) + 1, - parentId: existingMatch.id, - supersedes: [ - existingMatch.id, - ...(existingMatch.supersedes || []), - ], - sourceObservationIds: [ - ...new Set([...(existingMatch.sourceObservationIds ?? []), ...obsIds]), - ], - isLatest: true, - ...(scopedProject !== undefined && { project: scopedProject }), - }; - await kv.set(KV.memories, evolved.id, evolved); - existingMatch.isLatest = false; - await kv.set(KV.memories, existingMatch.id, existingMatch); - existingMemories.push(evolved); - writtenTitles.add(evolved.title.toLowerCase()); - deindexSupersededMemory(existingMatch.id); - await indexNewMemoryVersion(evolved); - await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], { - action: "mark_non_latest", - concept, - }); - await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], { - action: "evolve_memory", - oldId: existingMatch.id, - newId: evolved.id, - concept, - }); - consolidated++; - } else { - const memory: Memory = { - id: generateId("mem"), - createdAt: now, - updatedAt: now, - ...parsed, - sourceObservationIds: obsIds, - version: 1, - isLatest: true, - ...(scopedProject !== undefined && { project: scopedProject }), - }; - await kv.set(KV.memories, memory.id, memory); - existingMemories.push(memory); - writtenTitles.add(memory.title.toLowerCase()); - await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { - action: "create_memory", - concept, - }); - consolidated++; - } + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + if (existingMatch) { + const current = await kv.get(KV.memories, existingMatch.id); + if (!current || !evolvable(current) || current.version !== existingMatch.version || current.updatedAt !== existingMatch.updatedAt) return; + const evolved: Memory = { + id: generateId("mem"), + createdAt: now, + updatedAt: now, + ...parsed, + version: (current.version || 1) + 1, + parentId: existingMatch.id, + supersedes: [ + existingMatch.id, + ...(current.supersedes || []), + ], + sourceObservationIds: [ + ...new Set([...(current.sourceObservationIds ?? []), ...obsIds]), + ], + isLatest: true, + ...(scopedProject !== undefined && { project: scopedProject }), + }; + await kv.set(KV.memories, evolved.id, evolved); + existingMatch.isLatest = false; + await kv.set(KV.memories, current.id, { ...current, isLatest: false }); + existingMemories.push(evolved); + writtenTitles.add(evolved.title.toLowerCase()); + deindexSupersededMemory(existingMatch.id); + await indexNewMemoryVersion(evolved); + await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], { + action: "mark_non_latest", + concept, + }); + await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], { + action: "evolve_memory", + oldId: existingMatch.id, + newId: evolved.id, + concept, + }); + consolidated++; + } else { + const memory: Memory = { + id: generateId("mem"), + createdAt: now, + updatedAt: now, + ...parsed, + sourceObservationIds: obsIds, + version: 1, + isLatest: true, + ...(scopedProject !== undefined && { project: scopedProject }), + }; + await kv.set(KV.memories, memory.id, memory); + existingMemories.push(memory); + writtenTitles.add(memory.title.toLowerCase()); + await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { + action: "create_memory", + concept, + }); + consolidated++; + } + }); } catch (err) { logger.warn("Consolidation failed for concept", { concept, diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 0450a7d963..b4067969f7 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -9,6 +9,7 @@ import type { import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { isConsolidationEnabled, isGraphExtractionEnabled } from "../config.js"; import { graphLegDisabled } from "../state/graph-indexes.js"; import { recordAudit } from "./audit.js"; @@ -359,7 +360,21 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { evictedMemIds.add(mem.id); } else { try { - await deleteIndexed(kv, KV.memories, mem.id); + const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return false; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => { + logger.warn("Eviction could not release image reference", { + id: mem.id, + error: err instanceof Error ? err.message : String(err), + }); + }); + } + return true; + }); + if (!removed) continue; stats.expiredMemories++; evictedMemIds.add(mem.id); evictedSourceIds.push(mem.id); @@ -372,9 +387,6 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { }); continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "expired_memory", @@ -395,7 +407,21 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { stats.nonLatestMemories++; } else { try { - await deleteIndexed(kv, KV.memories, mem.id); + const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return false; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => { + logger.warn("Eviction could not release image reference", { + id: mem.id, + error: err instanceof Error ? err.message : String(err), + }); + }); + } + return true; + }); + if (!removed) continue; stats.nonLatestMemories++; evictedSourceIds.push(mem.id); } catch (err) { @@ -407,9 +433,6 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { }); continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "old_non_latest_memory", diff --git a/src/functions/governance.ts b/src/functions/governance.ts index 157ebeef34..af7afc777a 100644 --- a/src/functions/governance.ts +++ b/src/functions/governance.ts @@ -4,6 +4,8 @@ import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { recordAudit, safeAudit, queryAudit } from "./audit.js"; import { deleteIndexed } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { evictGraphForSources } from "./graph-eviction.js"; import { logger } from "../logger.js"; import { NOT_A_MEMORY_HINT } from "../mcp/tools-registry.js"; @@ -24,18 +26,20 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void { const deletedIds: string[] = []; const notFound: string[] = []; for (const id of data.memoryIds) { - const mem = await kv.get(KV.memories, id); - if (!mem) { - notFound.push(id); - } else { - await deleteIndexed(kv, KV.memories, id); - // The same teardown mem::forget does, so the two single-item - // delete paths cannot diverge in what they release. - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const mem = await kv.get(KV.memories, id); + if (!mem) { + notFound.push(id); + } else { + await deleteIndexed(kv, KV.memories, id); + // The same teardown mem::forget does, so the two single-item + // delete paths cannot diverge in what they release. + if (mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef); + } + deletedIds.push(id); } - deletedIds.push(id); - } + }); } const deleted = deletedIds.length; @@ -126,9 +130,11 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void { for (let i = 0; i < candidates.length; i += BATCH_SIZE) { const batch = candidates.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( - batch.map(async (mem) => { - await deleteIndexed(kv, KV.memories, mem.id); - }), + batch.map((mem) => + withKeyedLock(MEMORY_WRITE_LOCK, () => + deleteIndexed(kv, KV.memories, mem.id), + ), + ), ); results.forEach((result, j) => { const mem = batch[j]; diff --git a/src/functions/relations.ts b/src/functions/relations.ts index be23e3c8fb..b0d87c37c8 100644 --- a/src/functions/relations.ts +++ b/src/functions/relations.ts @@ -3,6 +3,7 @@ import type { Memory, MemoryRelation } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { safeAudit } from "./audit.js"; import { scrubFields } from "./privacy.js"; import { recordAccessBatch } from "./access-tracker.js"; @@ -46,11 +47,7 @@ export function registerRelationsFunction(sdk: ISdk, kv: StateKV): void { type: MemoryRelation["type"]; confidence?: number; }) => { - const [firstId, secondId] = [data.sourceId, data.targetId].sort(); - const lockKey = - firstId === secondId ? `mem:${firstId}` : `mem:${firstId}:${secondId}`; - - return withKeyedLock(lockKey, async () => { + return withKeyedLock(MEMORY_WRITE_LOCK, async () => { const source = await kv.get(KV.memories, data.sourceId); const target = await kv.get(KV.memories, data.targetId); if (!source || !target) { @@ -135,64 +132,66 @@ export function registerRelationsFunction(sdk: ISdk, kv: StateKV): void { newTitle?: string; }) => { data = scrubFields(data, "newContent", "newTitle"); - const existing = await kv.get(KV.memories, data.memoryId); - if (!existing) { - return { success: false, error: "memory not found" }; - } + return withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const existing = await kv.get(KV.memories, data.memoryId); + if (!existing) { + return { success: false, error: "memory not found" }; + } - const now = new Date().toISOString(); - const evolved: Memory = { - ...existing, - id: generateId("mem"), - createdAt: now, - updatedAt: now, - title: data.newTitle || existing.title, - content: data.newContent, - version: (existing.version || 1) + 1, - parentId: existing.id, - supersedes: [existing.id, ...(existing.supersedes || [])], - isLatest: true, - }; - - existing.isLatest = false; - await kv.set(KV.memories, existing.id, existing); - await safeAudit(kv, "evolve", "mem::evolve", [existing.id], { - operation: "evolve", - action: "mark_non_latest", - newId: evolved.id, - }); + const now = new Date().toISOString(); + const evolved: Memory = { + ...existing, + id: generateId("mem"), + createdAt: now, + updatedAt: now, + title: data.newTitle || existing.title, + content: data.newContent, + version: (existing.version || 1) + 1, + parentId: existing.id, + supersedes: [existing.id, ...(existing.supersedes || [])], + isLatest: true, + }; - deindexSupersededMemory(existing.id); - await kv.set(KV.memories, evolved.id, evolved); - await indexNewMemoryVersion(evolved); - await safeAudit(kv, "evolve", "mem::evolve", [evolved.id], { - operation: "evolve", - oldId: existing.id, - newId: evolved.id, - version: evolved.version, - }); + existing.isLatest = false; + await kv.set(KV.memories, existing.id, existing); + await safeAudit(kv, "evolve", "mem::evolve", [existing.id], { + operation: "evolve", + action: "mark_non_latest", + newId: evolved.id, + }); - const relation: MemoryRelation = { - type: "supersedes", - sourceId: evolved.id, - targetId: existing.id, - createdAt: now, - confidence: 1.0, - }; - const relationId = generateId("rel"); - await kv.set(KV.relations, relationId, relation); - await safeAudit(kv, "evolve", "mem::evolve", [relationId], { - operation: "supersedes", - oldId: existing.id, - newId: evolved.id, - }); + deindexSupersededMemory(existing.id); + await kv.set(KV.memories, evolved.id, evolved); + await indexNewMemoryVersion(evolved); + await safeAudit(kv, "evolve", "mem::evolve", [evolved.id], { + operation: "evolve", + oldId: existing.id, + newId: evolved.id, + version: evolved.version, + }); + + const relation: MemoryRelation = { + type: "supersedes", + sourceId: evolved.id, + targetId: existing.id, + createdAt: now, + confidence: 1.0, + }; + const relationId = generateId("rel"); + await kv.set(KV.relations, relationId, relation); + await safeAudit(kv, "evolve", "mem::evolve", [relationId], { + operation: "supersedes", + oldId: existing.id, + newId: evolved.id, + }); - logger.info("Memory evolved", { - oldId: existing.id, - newId: evolved.id, - version: evolved.version, + logger.info("Memory evolved", { + oldId: existing.id, + newId: evolved.id, + version: evolved.version, + }); + return { success: true, memory: evolved, previousId: existing.id }; }); - return { success: true, memory: evolved, previousId: existing.id }; }, ); diff --git a/src/functions/remember.ts b/src/functions/remember.ts index adb1364189..20520a995b 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -9,6 +9,7 @@ import { memoryToObservation, refersToDifferentDates, MEMORY_SESSION, + MEMORY_WRITE_LOCK, } from "../state/memory-utils.js"; import { recordAudit } from "./audit.js"; import { getSearchIndex, isMemoryIndexReady, vectorIndexAddBatchGuarded, deindexSupersededMemory, deleteIndexed } from "./search.js"; @@ -130,7 +131,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { return { success: false, error: "a Memory cannot have both a project and global" }; } - const saved = await withKeyedLock("mem:remember", async () => { + const saved = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { // Candidate generation: query the BM25 index with the new content // and Jaccard-compare only the top hits, instead of walking the // full memory corpus on every save. The index receives every @@ -392,15 +393,18 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { const { decrementImageRef } = await import("./image-refs.js"); if (data.memoryId) { - const mem = await kv.get(KV.memories, data.memoryId); - if (mem) { - await deleteIndexed(kv, KV.memories, data.memoryId); - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + const memoryId = data.memoryId; + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const mem = await kv.get(KV.memories, memoryId); + if (mem) { + await deleteIndexed(kv, KV.memories, memoryId); + if (mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef); + } + deletedMemoryIds.push(memoryId); + deleted++; } - deletedMemoryIds.push(data.memoryId); - deleted++; - } + }); } if ( diff --git a/src/functions/retention.ts b/src/functions/retention.ts index 1f071b3ebd..0b9b86d2f7 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -11,6 +11,8 @@ import type { AccessLog } from "./access-tracker.js"; import { emptyAccessLog, normalizeAccessLog } from "./access-tracker.js"; import { recordAudit } from "./audit.js"; import { deleteIndexed } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { evictGraphForSources } from "./graph-eviction.js"; import { logger } from "../logger.js"; @@ -345,13 +347,18 @@ export function registerRetentionFunctions( continue; } - const mem = await kv.get(scope, candidate.memoryId); - await deleteIndexed(kv, scope, candidate.memoryId); - if (mem && mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { - logger.warn("Retention could not release image reference", { memoryId: mem.id, error: String(error) }); - }); - } + const evictScope = scope; + const remove = async () => { + const mem = await kv.get(evictScope, candidate.memoryId); + await deleteIndexed(kv, evictScope, candidate.memoryId); + if (mem && mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { + logger.warn("Retention could not release image reference", { memoryId: mem.id, error: String(error) }); + }); + } + }; + if (evictScope === KV.memories) await withKeyedLock(MEMORY_WRITE_LOCK, remove); + else await remove(); await kv.delete(KV.retentionScores, candidate.memoryId); evicted++; evictedIds.push(candidate.memoryId); diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index 8931b2f047..afdc06d391 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -15,6 +15,8 @@ export class CircuitBreaker { private failures = 0; private lastFailureAt: number | null = null; private openedAt: number | null = null; + private probeInFlight = false; + private generation = 0; private readonly failureThreshold: number; private readonly failureWindowMs: number; @@ -31,33 +33,48 @@ export class CircuitBreaker { get isAllowed(): boolean { if (this.state === "closed") return true; + const now = Date.now(); if (this.state === "open") { - if ( - this.openedAt && - Date.now() - this.openedAt >= this.recoveryTimeoutMs - ) { - this.state = "half-open"; - return true; + if (this.openedAt === null || now - this.openedAt < this.recoveryTimeoutMs) { + return false; } - return false; + this.state = "half-open"; + this.probeInFlight = false; } + if (this.probeInFlight) return false; + this.probeInFlight = true; + this.generation++; return true; } - recordSuccess(): void { + acquire(): number | null { + return this.isAllowed ? this.generation : null; + } + + releaseProbe(generation = this.generation): void { + if (generation === this.generation && this.state === "half-open") this.probeInFlight = false; + } + + recordSuccess(generation = this.generation): void { + if (generation !== this.generation) return; if (this.state === "half-open") { this.state = "closed"; + this.generation++; this.failures = 0; this.lastFailureAt = null; this.openedAt = null; + this.probeInFlight = false; } } - recordFailure(): void { + recordFailure(generation = this.generation): void { + if (generation !== this.generation || this.state === "open") return; const now = Date.now(); if (this.state === "half-open") { this.state = "open"; + this.generation++; this.openedAt = now; + this.probeInFlight = false; return; } if (this.lastFailureAt && now - this.lastFailureAt > this.failureWindowMs) { @@ -67,6 +84,7 @@ export class CircuitBreaker { this.lastFailureAt = now; if (this.failures >= this.failureThreshold) { this.state = "open"; + this.generation++; this.openedAt = now; } } diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 976bea6ee5..d1ceee7bd6 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -72,19 +72,21 @@ export class ResilientProvider implements MemoryProvider { fn: () => Promise, ): Promise { const breaker = this.breakers[operation]; - if (!breaker.isAllowed) { - throw new Error("circuit_breaker_open"); - } + if (breaker.getState().state === "open") throw new Error("circuit_breaker_open"); await this.acquireSlot(); try { - const result = await fn(); - breaker.recordSuccess(); - return result; - } catch (err) { - // A provider that says "busy" or filtered one prompt is healthy; opening - // the breaker on it would fail unrelated work for the whole cooldown. - if (!isProviderBusy(err) && !isContentFilterRejection(err)) breaker.recordFailure(); - throw err; + const generation = breaker.acquire(); + if (generation === null) throw new Error("circuit_breaker_open"); + try { + const result = await fn(); + breaker.recordSuccess(generation); + return result; + } catch (err) { + // A busy or filtered prompt does not mean the provider is down. + if (!isProviderBusy(err) && !isContentFilterRejection(err)) breaker.recordFailure(generation); + else breaker.releaseProbe(generation); + throw err; + } } finally { this.releaseSlot(); } diff --git a/src/state/memory-utils.ts b/src/state/memory-utils.ts index 2a4e3996dd..10d8940bf7 100644 --- a/src/state/memory-utils.ts +++ b/src/state/memory-utils.ts @@ -166,3 +166,5 @@ export function memoryChunkJobs( text: header + "\n\n" + chunk, })); } + +export const MEMORY_WRITE_LOCK = "mem:remember"; diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index 21668fe8f6..9b0eada0ff 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -115,3 +115,77 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("half-open"); }); }); + +describe("CircuitBreaker half-open probe", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function openThenRecover(): CircuitBreaker { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + return cb; + } + + it("lets one caller probe while half-open and holds the rest", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + expect(cb.isAllowed).toBe(false); + }); + + it("closes after the probe succeeds and lets everyone through", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + cb.recordSuccess(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(true); + }); + + it("reopens with a fresh window after the probe fails", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + cb.recordFailure(); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + }); + + it("does not replace a pending probe after the recovery timeout", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(60_000); + expect(cb.isAllowed).toBe(false); + cb.releaseProbe(); + expect(cb.isAllowed).toBe(true); + }); +}); + +describe("CircuitBreaker probe ownership", () => { + afterEach(() => vi.useRealTimers()); + + it("ignores a stale probe completion after another probe starts", () => { + vi.useFakeTimers(); + const cb = new CircuitBreaker({ failureThreshold: 1 }); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + const first = cb.acquire()!; + cb.releaseProbe(first); + const second = cb.acquire()!; + cb.recordSuccess(first); + cb.recordFailure(first); + cb.releaseProbe(first); + expect(cb.getState().state).toBe("half-open"); + expect(cb.acquire()).toBeNull(); + cb.recordSuccess(second); + expect(cb.getState().state).toBe("closed"); + }); +}); diff --git a/test/consolidate-dedup.test.ts b/test/consolidate-dedup.test.ts index aa9c689d5d..8fd3b2e3ed 100644 --- a/test/consolidate-dedup.test.ts +++ b/test/consolidate-dedup.test.ts @@ -6,6 +6,9 @@ vi.mock("../src/logger.js", () => ({ import { registerConsolidateFunction } from "../src/functions/consolidate.js"; import { KV } from "../src/state/schema.js"; +import { StateKV } from "../src/state/kv.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../src/state/memory-utils.js"; import { getSearchIndex } from "../src/functions/search.js"; import { memoryToIndexDoc } from "../src/state/memory-utils.js"; import type { CompressedObservation, Memory, MemoryProvider, Session } from "../src/types.js"; @@ -94,6 +97,58 @@ async function setup(concepts: string[][]) { } describe("mem::consolidate dedup", () => { + it.each(["delete", "supersede", "update"])("skips a predecessor changed by %s during the model call", async (change) => { + const backing = makeMockKV(); + const sdk = makeMockSdk(); + const stateSdk = { + trigger: async ({ function_id, payload }: { function_id: string; payload: { scope: string; key: string; value: unknown } }) => { + if (function_id === "state::get") return backing.get(payload.scope, payload.key); + if (function_id === "state::set") return backing.set(payload.scope, payload.key, payload.value); + if (function_id === "state::delete") return backing.delete(payload.scope, payload.key); + if (function_id === "state::list") return backing.list(payload.scope); + throw new Error(`Unexpected function ${function_id}`); + }, + }; + const kv = new StateKV(stateSdk as never); + await kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) await kv.set(KV.observations("s1"), `o${i}`, obs(`o${i}`, ["auth"])); + const first: Memory = { + id: "first", title: "old", content: "old content", type: "pattern", concepts: [], files: [], + sessionIds: ["s1"], strength: 7, version: 1, isLatest: true, project: "api", + createdAt: "2026-01-01", updatedAt: "2026-01-01", sourceObservationIds: ["o0"], + }; + await kv.set(KV.memories, first.id, first); + getSearchIndex().clear(); + getSearchIndex().add(memoryToIndexDoc(first), "memory"); + let release!: (response: string) => void; + let started!: () => void; + const entered = new Promise((resolve) => { started = resolve; }); + const provider = rewordingProvider(); + (provider.compress as ReturnType).mockImplementation(() => { + started(); + return new Promise((resolve) => { release = resolve; }); + }); + registerConsolidateFunction(sdk as never, kv, provider); + const running = sdk.trigger("mem::consolidate", { project: "api", minObservations: 1 }); + await entered; + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + if (change === "delete") { + await kv.delete(KV.memories, first.id); + getSearchIndex().remove(first.id); + } else if (change === "supersede") { + await kv.set(KV.memories, first.id, { ...first, isLatest: false }); + } else { + await kv.set(KV.memories, first.id, { ...first, updatedAt: "2026-02-01", content: "updated" }); + } + }); + release("patternnewnew content"); + expect(await running).toMatchObject({ consolidated: 0 }); + const memories = await kv.list(KV.memories); + expect(memories).toHaveLength(change === "delete" ? 0 : 1); + if (change === "supersede") expect(memories[0].isLatest).toBe(false); + if (change === "update") expect(memories[0].content).toBe("updated"); + if (change === "delete") expect(getSearchIndex().has(first.id)).toBe(false); + }); it("writes nothing on a second run over unchanged input", async () => { const { run, kv } = await setup([["auth"], ["auth"], ["auth"]]); expect(await run()).toMatchObject({ consolidated: 1 }); diff --git a/test/evict.test.ts b/test/evict.test.ts index b4a8b43364..53f241e22c 100644 --- a/test/evict.test.ts +++ b/test/evict.test.ts @@ -708,7 +708,7 @@ describe("mem::evict default importance floor", () => { .sort(); const keptMem = async () => (await kv.list(KV.memories)).map((m) => m.id).sort(); - return { run, keptObs, keptMem }; + return { run, keptObs, keptMem, kv }; } it("removes a one-day-old Observation of importance 4 and keeps importance 5 and a fresh one", async () => { @@ -737,6 +737,25 @@ describe("mem::evict default importance floor", () => { expect(await t.keptMem()).toEqual(["latest", "young"]); }); + it.each([ + ["expiredMemories", "expired_memory", { ...memory("old", 100, true), forgetAfter: daysAgo(1) }], + ["nonLatestMemories", "old_non_latest_memory", memory("old", 100, false)], + ] as const)("counts and audits %s when image release fails", async (counter, reason, old) => { + const t = setup([], [{ ...old, imageRef: "images/old.png" }]); + const get = t.kv.get; + vi.spyOn(t.kv, "get").mockImplementation((scope, key) => { + if (scope === KV.imageRefs) throw new Error("image refs unavailable"); + return get(scope, key); + }); + + expect(await t.run()).toMatchObject({ [counter]: 1 }); + expect(await t.keptMem()).toEqual([]); + expect(await t.kv.list(KV.audit)).toContainEqual(expect.objectContaining({ + operation: "delete", + details: expect.objectContaining({ reason }), + })); + }); + it("reports the default removals on a dry run without deleting", async () => { const t = setup( [obs("ses_d", "low", 1, 4), obs("ses_d", "ok", 1, 5)], diff --git a/test/memory-write-locks.test.ts b/test/memory-write-locks.test.ts new file mode 100644 index 0000000000..17329175c8 --- /dev/null +++ b/test/memory-write-locks.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerRememberFunction } from "../src/functions/remember.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerGovernanceFunction } from "../src/functions/governance.js"; +import { registerAutoForgetFunction } from "../src/functions/auto-forget.js"; +import { registerEvictFunction } from "../src/functions/evict.js"; +import { registerRetentionFunctions } from "../src/functions/retention.js"; +import type { Memory } from "../src/types.js"; + +const MEMORIES = "mem:memories"; +const tick = () => new Promise((r) => setTimeout(r, 15)); + +function gatedKV() { + const store = new Map>(); + let gate: { scope: string; key: string; reached: () => void; open: Promise } | null = null; + const clone = (v: T): T => structuredClone(v); + const kv = { + get: async (scope: string, key: string): Promise => { + const row = store.get(scope)?.get(key); + return row === undefined ? null : clone(row as T); + }, + set: async (scope: string, key: string, data: T): Promise => { + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + g.reached(); + await g.open; + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, clone(data)); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []).map((v) => clone(v as T)), + }; + function pauseNextSet(scope: string, key: string) { + let release!: () => void; + let reached!: () => void; + const open = new Promise((r) => (release = r)); + const hit = new Promise((r) => (reached = r)); + gate = { scope, key, reached, open }; + return { release, hit }; + } + return { kv, pauseNextSet }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + functions.set(typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, handler); + }, + registerTrigger: () => {}, + trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +function makeMemory(overrides: Partial = {}): Memory { + return { + id: "mem_1", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + type: "fact", + title: "Lock probe", + content: "zebra quartz lattice meridian harbor", + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: true, + ...overrides, + }; +} + +describe("Memory writers serialize with deletes and relates", () => { + let sdk: ReturnType; + let env: ReturnType; + + beforeEach(() => { + sdk = mockSdk(); + env = gatedKV(); + registerRememberFunction(sdk as never, env.kv as never); + registerRelationsFunction(sdk as never, env.kv as never); + registerGovernanceFunction(sdk as never, env.kv as never); + registerAutoForgetFunction(sdk as never, env.kv as never); + registerEvictFunction(sdk as never, env.kv as never); + registerRetentionFunctions(sdk as never, env.kv as never); + }); + + async function deleteDuringReinforce( + seed: Partial, + del: () => Promise, + ) { + const x = makeMemory({ id: "mem_x", ...seed }); + await env.kv.set(MEMORIES, x.id, x); + const gate = env.pauseNextSet(MEMORIES, x.id); + const save = sdk.trigger("mem::remember", { content: x.content }); + await gate.hit; + const deletion = del(); + await tick(); + gate.release(); + await Promise.all([save, deletion]); + return env.kv.get(MEMORIES, x.id); + } + + it("governance-delete is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::governance-delete", { memoryIds: ["mem_x"] }), + ); + expect(left).toBeNull(); + }); + + it("governance-bulk is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::governance-bulk", { type: ["fact"] }), + ); + expect(left).toBeNull(); + }); + + it("forget is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::forget", { memoryId: "mem_x" }), + ); + expect(left).toBeNull(); + }); + + it("auto-forget TTL is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({ forgetAfter: "2020-01-01T00:00:00Z" }, () => + sdk.trigger("mem::auto-forget", {}), + ); + expect(left).toBeNull(); + }); + + it("evict TTL is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({ forgetAfter: "2020-01-01T00:00:00Z" }, () => + sdk.trigger("mem::evict", {}), + ); + expect(left).toBeNull(); + }); + + it("retention-evict is not undone by a remember that read the Memory first", async () => { + await env.kv.set("mem:retention", "mem_x", { + memoryId: "mem_x", + score: 0, + source: "episodic", + }); + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::retention-evict", { threshold: 1 }), + ); + expect(left).toBeNull(); + }); + + it("two relates sharing a Memory keep both relatedIds", async () => { + for (const id of ["mem_a", "mem_b", "mem_c"]) { + await env.kv.set(MEMORIES, id, makeMemory({ id, content: `${id} unrelated text` })); + } + await Promise.all([ + sdk.trigger("mem::relate", { sourceId: "mem_a", targetId: "mem_b", type: "related" }), + sdk.trigger("mem::relate", { sourceId: "mem_a", targetId: "mem_c", type: "related" }), + ]); + const a = await env.kv.get(MEMORIES, "mem_a"); + expect([...(a!.relatedIds ?? [])].sort()).toEqual(["mem_b", "mem_c"]); + }); + + it("a relate that read a Memory before evolve does not mark the superseded version latest", async () => { + await env.kv.set(MEMORIES, "mem_a", makeMemory({ id: "mem_a", content: "alpha record" })); + await env.kv.set(MEMORIES, "mem_b", makeMemory({ id: "mem_b", content: "beta record" })); + const gate = env.pauseNextSet(MEMORIES, "mem_a"); + const relate = sdk.trigger("mem::relate", { + sourceId: "mem_a", + targetId: "mem_b", + type: "related", + }); + await gate.hit; + const evolve = sdk.trigger("mem::evolve", { memoryId: "mem_a", newContent: "alpha record v2" }); + await tick(); + gate.release(); + await Promise.all([relate, evolve]); + const a = await env.kv.get(MEMORIES, "mem_a"); + expect(a!.isLatest).toBe(false); + }); +}); diff --git a/test/resilient-half-open.test.ts b/test/resilient-half-open.test.ts new file mode 100644 index 0000000000..f9061c43bb --- /dev/null +++ b/test/resilient-half-open.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +describe("ResilientProvider half-open breaker", () => { + afterEach(() => vi.useRealTimers()); + + it("sends one probe to a provider that is still down, not every queued call", async () => { + vi.useFakeTimers(); + let calls = 0; + const inner: MemoryProvider = { + name: "down", + compress: async () => { + calls++; + throw new Error("connection refused"); + }, + summarize: async () => "", + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + expect(calls).toBe(3); + + vi.advanceTimersByTime(30_000); + calls = 0; + await Promise.allSettled( + Array.from({ length: 8 }, () => provider.compress("s", "u")), + ); + expect(calls).toBe(1); + }); + it.each([ + ["busy", Object.assign(new Error("busy"), { status: 429 })], + ["filtered", new Error("content_filter")], + ])("releases a %s probe for the next call", async (_name, rejection) => { + vi.useFakeTimers(); + let mode = "down"; + const inner: MemoryProvider = { + name: "synthetic", + compress: async () => { + if (mode === "down") throw new Error("down"); + if (mode === "reject") throw rejection; + return "ok"; + }, + summarize: async () => "", + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + vi.advanceTimersByTime(30_000); + mode = "reject"; + await expect(provider.compress("s", "u")).rejects.toThrow(); + mode = "ok"; + await expect(provider.compress("s", "u")).resolves.toBe("ok"); + }); + + it("rejects an open circuit without waiting for a shared slot", async () => { + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + const inner: MemoryProvider = { + name: "synthetic", + compress: async () => { throw new Error("down"); }, + summarize: async () => { await pending; return "ok"; }, + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + const slots = Array.from({ length: 4 }, () => provider.summarize("s", "u")); + await Promise.resolve(); + try { + await expect(provider.compress("s", "u")).rejects.toThrow("circuit_breaker_open"); + } finally { + release(); + await Promise.all(slots); + } + }); + +});