diff --git a/CLAUDE.md b/CLAUDE.md index 5268a1043d..ed70cf3ed4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -237,6 +237,18 @@ Auto-forget and retention release a Memory's image reference after its indexed d A failed release is logged without blocking deletion bookkeeping; it may leave an unused image. Concurrent deletion locking is outside this contract. +### Memory writes and provider recovery +`MEMORY_WRITE_LOCK` (`src/state/memory-utils.ts`) serializes Memory saves, deletes, +relations, evolution, and auto-forget demotion. Before demoting, auto-forget rechecks both +live rows under the lock and reports only completed demotions; dry runs report snapshot matches. +Consolidation checks the predecessor again under the lock after the provider call. Release the +lock before graph eviction. Import and snapshot restore keep their own replacement semantics. +Eviction decrements image references after deleting the row and logs a failed release without +skipping bookkeeping. +`ResilientProvider` admits one half-open probe per operation after acquiring a provider slot. +The reservation lasts until the call settles; busy and content-filter responses release it, +and probe generations discard stale completions. An open circuit rejects before waiting for a slot. + ### Replace import `mem::import` validates section shapes, size limits and keyed fields used by import indexing before any store mutation. Rows without string keys are dropped and counted in `invalid`, so @@ -469,7 +481,7 @@ short forms count only beside an ordinal, a year, or after "on". The in-process BM25 from content at boot, so there is no migration. ### Search and consolidation -`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; cross-writer snapshot races are not covered by this lock. +`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; the write lock does not make both writes transactional. ### Search on the event loop Search runs on the daemon's one event loop, which also answers `/livez` and the hooks. BM25 diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts index ccf45da817..3479726c25 100644 --- a/src/functions/auto-forget.ts +++ b/src/functions/auto-forget.ts @@ -4,9 +4,10 @@ import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { recordAudit } from "./audit.js"; import { deleteIndexed, deindexSupersededMemory } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { lowerObservationCounts } from "./observe.js"; import { evictGraphForSources } from "./graph-eviction.js"; -import { refersToDifferentDates } from "../state/memory-utils.js"; +import { refersToDifferentDates, MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { logger } from "../logger.js"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -18,6 +19,22 @@ function sameSupersessionScope(a: Memory, b: Memory): boolean { a.agentId === b.agentId; } +function contradictionTokens(content: string): Set { + return new Set(content.toLowerCase().split(/\s+/).filter((word) => word.length > 2)); +} + +function contradictionSimilarity(a: Memory, b: Memory, setA: Set, setB: Set): number { + if (!setA.size || !setB.size) return 0; + let intersection = 0; + for (const word of setA) { + if (setB.has(word)) intersection++; + } + const similarity = intersection / (setA.size + setB.size - intersection); + // Daily reports for different dates are not contradictions, even if their text nearly matches. + return similarity > CONTRADICTION_THRESHOLD && sameSupersessionScope(a, b) && + !refersToDifferentDates(a.content, b.content) ? similarity : 0; +} + interface AutoForgetResult { ttlExpired: string[]; contradictions: Array<{ @@ -53,12 +70,16 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void { result.ttlExpired.push(mem.id); deletedIds.add(mem.id); if (!dryRun) { - await deleteIndexed(kv, KV.memories, mem.id); - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { - logger.warn("Auto-forget could not release image reference", { memoryId: mem.id, error: String(error) }); - }); - } + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((error) => { + logger.warn("Auto-forget could not release image reference", { memoryId: current.id, error: String(error) }); + }); + } + }); evictedSourceIds.push(mem.id); await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], { resource: "memory", @@ -78,19 +99,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void { ) .slice(0, 1000); - const tokenCache = new Map>(); - for (const mem of latestMemories) { - tokenCache.set( - mem.id, - new Set( - mem.content - .toLowerCase() - .split(/\s+/) - .filter((t) => t.length > 2), - ), - ); - } - + const tokenCache = new Map(latestMemories.map((mem) => [mem.id, contradictionTokens(mem.content)])); const memById = new Map(latestMemories.map((m) => [m.id, m])); const conceptIndex = new Map(); for (const mem of latestMemories) { @@ -113,53 +122,38 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void { if (compared.has(key)) continue; compared.add(key); - const setA = tokenCache.get(memIds[i])!; - const setB = tokenCache.get(memIds[j])!; - let intersection = 0; - if (setA.size === 0 && setB.size === 0) continue; - if (setA.size === 0 || setB.size === 0) continue; - for (const word of setA) { - if (setB.has(word)) intersection++; + const memA = memById.get(memIds[i])!; + const memB = memById.get(memIds[j])!; + if (memA.id === memB.id) continue; + const similarity = contradictionSimilarity(memA, memB, tokenCache.get(memA.id)!, tokenCache.get(memB.id)!); + if (!similarity) continue; + if (dryRun) { + result.contradictions.push({ memoryA: memA.id, memoryB: memB.id, similarity }); + continue; } - const sim = - intersection / (setA.size + setB.size - intersection); - - if (sim > CONTRADICTION_THRESHOLD) { - const memA = memById.get(memIds[i])!; - const memB = memById.get(memIds[j])!; - if (!sameSupersessionScope(memA, memB)) continue; - // Two daily reports of two different days are not a - // contradiction. Without this, auto-forget silently re-demotes - // exactly the rows a lineage repair just restored - measured - // 2026-08-10, one hour after the step 5 repair - and unlike - // remember it records NO supersedes edge, so a lineage census - // cannot even see what it did. - if (refersToDifferentDates(memA.content, memB.content)) { - continue; - } - result.contradictions.push({ - memoryA: memA.id, - memoryB: memB.id, - similarity: sim, - }); - if (!dryRun) { - const older = - new Date(memA.createdAt).getTime() < - new Date(memB.createdAt).getTime() - ? memA - : memB; - older.isLatest = false; - await kv.set(KV.memories, older.id, older); - deindexSupersededMemory(older.id); - await recordAudit(kv, "forget", "mem::auto-forget", [older.id], { - resource: "memory", - reason: "auto-forget contradiction", - olderId: older.id, - similarity: sim, - }); - } - } + const demotion = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const [a, b] = await Promise.all([ + kv.get(KV.memories, memA.id), + kv.get(KV.memories, memB.id), + ]); + if (!a || !b || a.isLatest === false || b.isLatest === false) return; + const conceptsB = new Set(b.concepts?.map((concept) => concept.toLowerCase())); + if (!a.concepts?.some((concept) => conceptsB.has(concept.toLowerCase()))) return; + const similarity = contradictionSimilarity(a, b, contradictionTokens(a.content), contradictionTokens(b.content)); + if (!similarity) return; + const older = new Date(a.createdAt).getTime() < new Date(b.createdAt).getTime() ? a : b; + await kv.set(KV.memories, older.id, { ...older, isLatest: false }); + deindexSupersededMemory(older.id); + return { olderId: older.id, similarity }; + }); + if (!demotion) continue; + result.contradictions.push({ memoryA: memA.id, memoryB: memB.id, similarity: demotion.similarity }); + await recordAudit(kv, "forget", "mem::auto-forget", [demotion.olderId], { + resource: "memory", + reason: "auto-forget contradiction", + ...demotion, + }); } } } diff --git a/src/functions/consolidate.ts b/src/functions/consolidate.ts index f2e431d140..debc0ff0e7 100644 --- a/src/functions/consolidate.ts +++ b/src/functions/consolidate.ts @@ -7,6 +7,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; import { deindexSupersededMemory, indexNewMemoryVersion } from "./search.js"; @@ -113,7 +114,7 @@ export function registerConsolidateFunction( } let consolidated = 0; - const existingMemories = await kv.list(KV.memories); + const existingMemories = structuredClone(await kv.list(KV.memories)); const writtenTitles = new Set(); const scopedProject = typeof data.project === "string" && data.project.trim().length > 0 @@ -185,62 +186,66 @@ export function registerConsolidateFunction( m.title.toLowerCase() === parsed.title.toLowerCase(), ); - if (existingMatch) { - const evolved: Memory = { - id: generateId("mem"), - createdAt: now, - updatedAt: now, - ...parsed, - version: (existingMatch.version || 1) + 1, - parentId: existingMatch.id, - supersedes: [ - existingMatch.id, - ...(existingMatch.supersedes || []), - ], - sourceObservationIds: [ - ...new Set([...(existingMatch.sourceObservationIds ?? []), ...obsIds]), - ], - isLatest: true, - ...(scopedProject !== undefined && { project: scopedProject }), - }; - await kv.set(KV.memories, evolved.id, evolved); - existingMatch.isLatest = false; - await kv.set(KV.memories, existingMatch.id, existingMatch); - existingMemories.push(evolved); - writtenTitles.add(evolved.title.toLowerCase()); - deindexSupersededMemory(existingMatch.id); - await indexNewMemoryVersion(evolved); - await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], { - action: "mark_non_latest", - concept, - }); - await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], { - action: "evolve_memory", - oldId: existingMatch.id, - newId: evolved.id, - concept, - }); - consolidated++; - } else { - const memory: Memory = { - id: generateId("mem"), - createdAt: now, - updatedAt: now, - ...parsed, - sourceObservationIds: obsIds, - version: 1, - isLatest: true, - ...(scopedProject !== undefined && { project: scopedProject }), - }; - await kv.set(KV.memories, memory.id, memory); - existingMemories.push(memory); - writtenTitles.add(memory.title.toLowerCase()); - await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { - action: "create_memory", - concept, - }); - consolidated++; - } + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + if (existingMatch) { + const current = await kv.get(KV.memories, existingMatch.id); + if (!current || !evolvable(current) || current.version !== existingMatch.version || current.updatedAt !== existingMatch.updatedAt) return; + const evolved: Memory = { + id: generateId("mem"), + createdAt: now, + updatedAt: now, + ...parsed, + version: (current.version || 1) + 1, + parentId: existingMatch.id, + supersedes: [ + existingMatch.id, + ...(current.supersedes || []), + ], + sourceObservationIds: [ + ...new Set([...(current.sourceObservationIds ?? []), ...obsIds]), + ], + isLatest: true, + ...(scopedProject !== undefined && { project: scopedProject }), + }; + await kv.set(KV.memories, evolved.id, evolved); + existingMatch.isLatest = false; + await kv.set(KV.memories, current.id, { ...current, isLatest: false }); + existingMemories.push(evolved); + writtenTitles.add(evolved.title.toLowerCase()); + deindexSupersededMemory(existingMatch.id); + await indexNewMemoryVersion(evolved); + await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], { + action: "mark_non_latest", + concept, + }); + await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], { + action: "evolve_memory", + oldId: existingMatch.id, + newId: evolved.id, + concept, + }); + consolidated++; + } else { + const memory: Memory = { + id: generateId("mem"), + createdAt: now, + updatedAt: now, + ...parsed, + sourceObservationIds: obsIds, + version: 1, + isLatest: true, + ...(scopedProject !== undefined && { project: scopedProject }), + }; + await kv.set(KV.memories, memory.id, memory); + existingMemories.push(memory); + writtenTitles.add(memory.title.toLowerCase()); + await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { + action: "create_memory", + concept, + }); + consolidated++; + } + }); } catch (err) { logger.warn("Consolidation failed for concept", { concept, diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 0450a7d963..b4067969f7 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -9,6 +9,7 @@ import type { import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { isConsolidationEnabled, isGraphExtractionEnabled } from "../config.js"; import { graphLegDisabled } from "../state/graph-indexes.js"; import { recordAudit } from "./audit.js"; @@ -359,7 +360,21 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { evictedMemIds.add(mem.id); } else { try { - await deleteIndexed(kv, KV.memories, mem.id); + const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return false; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => { + logger.warn("Eviction could not release image reference", { + id: mem.id, + error: err instanceof Error ? err.message : String(err), + }); + }); + } + return true; + }); + if (!removed) continue; stats.expiredMemories++; evictedMemIds.add(mem.id); evictedSourceIds.push(mem.id); @@ -372,9 +387,6 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { }); continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "expired_memory", @@ -395,7 +407,21 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { stats.nonLatestMemories++; } else { try { - await deleteIndexed(kv, KV.memories, mem.id); + const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const current = await kv.get(KV.memories, mem.id); + if (!current) return false; + await deleteIndexed(kv, KV.memories, mem.id); + if (current.imageRef) { + await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => { + logger.warn("Eviction could not release image reference", { + id: mem.id, + error: err instanceof Error ? err.message : String(err), + }); + }); + } + return true; + }); + if (!removed) continue; stats.nonLatestMemories++; evictedSourceIds.push(mem.id); } catch (err) { @@ -407,9 +433,6 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { }); continue; } - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); - } await recordAudit(kv, "delete", "mem::evict", [mem.id], { resource: "memory", reason: "old_non_latest_memory", diff --git a/src/functions/governance.ts b/src/functions/governance.ts index 157ebeef34..af7afc777a 100644 --- a/src/functions/governance.ts +++ b/src/functions/governance.ts @@ -4,6 +4,8 @@ import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { recordAudit, safeAudit, queryAudit } from "./audit.js"; import { deleteIndexed } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { evictGraphForSources } from "./graph-eviction.js"; import { logger } from "../logger.js"; import { NOT_A_MEMORY_HINT } from "../mcp/tools-registry.js"; @@ -24,18 +26,20 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void { const deletedIds: string[] = []; const notFound: string[] = []; for (const id of data.memoryIds) { - const mem = await kv.get(KV.memories, id); - if (!mem) { - notFound.push(id); - } else { - await deleteIndexed(kv, KV.memories, id); - // The same teardown mem::forget does, so the two single-item - // delete paths cannot diverge in what they release. - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const mem = await kv.get(KV.memories, id); + if (!mem) { + notFound.push(id); + } else { + await deleteIndexed(kv, KV.memories, id); + // The same teardown mem::forget does, so the two single-item + // delete paths cannot diverge in what they release. + if (mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef); + } + deletedIds.push(id); } - deletedIds.push(id); - } + }); } const deleted = deletedIds.length; @@ -126,9 +130,11 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void { for (let i = 0; i < candidates.length; i += BATCH_SIZE) { const batch = candidates.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( - batch.map(async (mem) => { - await deleteIndexed(kv, KV.memories, mem.id); - }), + batch.map((mem) => + withKeyedLock(MEMORY_WRITE_LOCK, () => + deleteIndexed(kv, KV.memories, mem.id), + ), + ), ); results.forEach((result, j) => { const mem = batch[j]; diff --git a/src/functions/relations.ts b/src/functions/relations.ts index be23e3c8fb..b0d87c37c8 100644 --- a/src/functions/relations.ts +++ b/src/functions/relations.ts @@ -3,6 +3,7 @@ import type { Memory, MemoryRelation } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { safeAudit } from "./audit.js"; import { scrubFields } from "./privacy.js"; import { recordAccessBatch } from "./access-tracker.js"; @@ -46,11 +47,7 @@ export function registerRelationsFunction(sdk: ISdk, kv: StateKV): void { type: MemoryRelation["type"]; confidence?: number; }) => { - const [firstId, secondId] = [data.sourceId, data.targetId].sort(); - const lockKey = - firstId === secondId ? `mem:${firstId}` : `mem:${firstId}:${secondId}`; - - return withKeyedLock(lockKey, async () => { + return withKeyedLock(MEMORY_WRITE_LOCK, async () => { const source = await kv.get(KV.memories, data.sourceId); const target = await kv.get(KV.memories, data.targetId); if (!source || !target) { @@ -135,64 +132,66 @@ export function registerRelationsFunction(sdk: ISdk, kv: StateKV): void { newTitle?: string; }) => { data = scrubFields(data, "newContent", "newTitle"); - const existing = await kv.get(KV.memories, data.memoryId); - if (!existing) { - return { success: false, error: "memory not found" }; - } + return withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const existing = await kv.get(KV.memories, data.memoryId); + if (!existing) { + return { success: false, error: "memory not found" }; + } - const now = new Date().toISOString(); - const evolved: Memory = { - ...existing, - id: generateId("mem"), - createdAt: now, - updatedAt: now, - title: data.newTitle || existing.title, - content: data.newContent, - version: (existing.version || 1) + 1, - parentId: existing.id, - supersedes: [existing.id, ...(existing.supersedes || [])], - isLatest: true, - }; - - existing.isLatest = false; - await kv.set(KV.memories, existing.id, existing); - await safeAudit(kv, "evolve", "mem::evolve", [existing.id], { - operation: "evolve", - action: "mark_non_latest", - newId: evolved.id, - }); + const now = new Date().toISOString(); + const evolved: Memory = { + ...existing, + id: generateId("mem"), + createdAt: now, + updatedAt: now, + title: data.newTitle || existing.title, + content: data.newContent, + version: (existing.version || 1) + 1, + parentId: existing.id, + supersedes: [existing.id, ...(existing.supersedes || [])], + isLatest: true, + }; - deindexSupersededMemory(existing.id); - await kv.set(KV.memories, evolved.id, evolved); - await indexNewMemoryVersion(evolved); - await safeAudit(kv, "evolve", "mem::evolve", [evolved.id], { - operation: "evolve", - oldId: existing.id, - newId: evolved.id, - version: evolved.version, - }); + existing.isLatest = false; + await kv.set(KV.memories, existing.id, existing); + await safeAudit(kv, "evolve", "mem::evolve", [existing.id], { + operation: "evolve", + action: "mark_non_latest", + newId: evolved.id, + }); - const relation: MemoryRelation = { - type: "supersedes", - sourceId: evolved.id, - targetId: existing.id, - createdAt: now, - confidence: 1.0, - }; - const relationId = generateId("rel"); - await kv.set(KV.relations, relationId, relation); - await safeAudit(kv, "evolve", "mem::evolve", [relationId], { - operation: "supersedes", - oldId: existing.id, - newId: evolved.id, - }); + deindexSupersededMemory(existing.id); + await kv.set(KV.memories, evolved.id, evolved); + await indexNewMemoryVersion(evolved); + await safeAudit(kv, "evolve", "mem::evolve", [evolved.id], { + operation: "evolve", + oldId: existing.id, + newId: evolved.id, + version: evolved.version, + }); + + const relation: MemoryRelation = { + type: "supersedes", + sourceId: evolved.id, + targetId: existing.id, + createdAt: now, + confidence: 1.0, + }; + const relationId = generateId("rel"); + await kv.set(KV.relations, relationId, relation); + await safeAudit(kv, "evolve", "mem::evolve", [relationId], { + operation: "supersedes", + oldId: existing.id, + newId: evolved.id, + }); - logger.info("Memory evolved", { - oldId: existing.id, - newId: evolved.id, - version: evolved.version, + logger.info("Memory evolved", { + oldId: existing.id, + newId: evolved.id, + version: evolved.version, + }); + return { success: true, memory: evolved, previousId: existing.id }; }); - return { success: true, memory: evolved, previousId: existing.id }; }, ); diff --git a/src/functions/remember.ts b/src/functions/remember.ts index adb1364189..20520a995b 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -9,6 +9,7 @@ import { memoryToObservation, refersToDifferentDates, MEMORY_SESSION, + MEMORY_WRITE_LOCK, } from "../state/memory-utils.js"; import { recordAudit } from "./audit.js"; import { getSearchIndex, isMemoryIndexReady, vectorIndexAddBatchGuarded, deindexSupersededMemory, deleteIndexed } from "./search.js"; @@ -130,7 +131,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { return { success: false, error: "a Memory cannot have both a project and global" }; } - const saved = await withKeyedLock("mem:remember", async () => { + const saved = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { // Candidate generation: query the BM25 index with the new content // and Jaccard-compare only the top hits, instead of walking the // full memory corpus on every save. The index receives every @@ -392,15 +393,18 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { const { decrementImageRef } = await import("./image-refs.js"); if (data.memoryId) { - const mem = await kv.get(KV.memories, data.memoryId); - if (mem) { - await deleteIndexed(kv, KV.memories, data.memoryId); - if (mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef); + const memoryId = data.memoryId; + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const mem = await kv.get(KV.memories, memoryId); + if (mem) { + await deleteIndexed(kv, KV.memories, memoryId); + if (mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef); + } + deletedMemoryIds.push(memoryId); + deleted++; } - deletedMemoryIds.push(data.memoryId); - deleted++; - } + }); } if ( diff --git a/src/functions/retention.ts b/src/functions/retention.ts index 1f071b3ebd..0b9b86d2f7 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -11,6 +11,8 @@ import type { AccessLog } from "./access-tracker.js"; import { emptyAccessLog, normalizeAccessLog } from "./access-tracker.js"; import { recordAudit } from "./audit.js"; import { deleteIndexed } from "./search.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { evictGraphForSources } from "./graph-eviction.js"; import { logger } from "../logger.js"; @@ -345,13 +347,18 @@ export function registerRetentionFunctions( continue; } - const mem = await kv.get(scope, candidate.memoryId); - await deleteIndexed(kv, scope, candidate.memoryId); - if (mem && mem.imageRef) { - await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { - logger.warn("Retention could not release image reference", { memoryId: mem.id, error: String(error) }); - }); - } + const evictScope = scope; + const remove = async () => { + const mem = await kv.get(evictScope, candidate.memoryId); + await deleteIndexed(kv, evictScope, candidate.memoryId); + if (mem && mem.imageRef) { + await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => { + logger.warn("Retention could not release image reference", { memoryId: mem.id, error: String(error) }); + }); + } + }; + if (evictScope === KV.memories) await withKeyedLock(MEMORY_WRITE_LOCK, remove); + else await remove(); await kv.delete(KV.retentionScores, candidate.memoryId); evicted++; evictedIds.push(candidate.memoryId); diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index 8931b2f047..afdc06d391 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -15,6 +15,8 @@ export class CircuitBreaker { private failures = 0; private lastFailureAt: number | null = null; private openedAt: number | null = null; + private probeInFlight = false; + private generation = 0; private readonly failureThreshold: number; private readonly failureWindowMs: number; @@ -31,33 +33,48 @@ export class CircuitBreaker { get isAllowed(): boolean { if (this.state === "closed") return true; + const now = Date.now(); if (this.state === "open") { - if ( - this.openedAt && - Date.now() - this.openedAt >= this.recoveryTimeoutMs - ) { - this.state = "half-open"; - return true; + if (this.openedAt === null || now - this.openedAt < this.recoveryTimeoutMs) { + return false; } - return false; + this.state = "half-open"; + this.probeInFlight = false; } + if (this.probeInFlight) return false; + this.probeInFlight = true; + this.generation++; return true; } - recordSuccess(): void { + acquire(): number | null { + return this.isAllowed ? this.generation : null; + } + + releaseProbe(generation = this.generation): void { + if (generation === this.generation && this.state === "half-open") this.probeInFlight = false; + } + + recordSuccess(generation = this.generation): void { + if (generation !== this.generation) return; if (this.state === "half-open") { this.state = "closed"; + this.generation++; this.failures = 0; this.lastFailureAt = null; this.openedAt = null; + this.probeInFlight = false; } } - recordFailure(): void { + recordFailure(generation = this.generation): void { + if (generation !== this.generation || this.state === "open") return; const now = Date.now(); if (this.state === "half-open") { this.state = "open"; + this.generation++; this.openedAt = now; + this.probeInFlight = false; return; } if (this.lastFailureAt && now - this.lastFailureAt > this.failureWindowMs) { @@ -67,6 +84,7 @@ export class CircuitBreaker { this.lastFailureAt = now; if (this.failures >= this.failureThreshold) { this.state = "open"; + this.generation++; this.openedAt = now; } } diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 976bea6ee5..d1ceee7bd6 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -72,19 +72,21 @@ export class ResilientProvider implements MemoryProvider { fn: () => Promise, ): Promise { const breaker = this.breakers[operation]; - if (!breaker.isAllowed) { - throw new Error("circuit_breaker_open"); - } + if (breaker.getState().state === "open") throw new Error("circuit_breaker_open"); await this.acquireSlot(); try { - const result = await fn(); - breaker.recordSuccess(); - return result; - } catch (err) { - // A provider that says "busy" or filtered one prompt is healthy; opening - // the breaker on it would fail unrelated work for the whole cooldown. - if (!isProviderBusy(err) && !isContentFilterRejection(err)) breaker.recordFailure(); - throw err; + const generation = breaker.acquire(); + if (generation === null) throw new Error("circuit_breaker_open"); + try { + const result = await fn(); + breaker.recordSuccess(generation); + return result; + } catch (err) { + // A busy or filtered prompt does not mean the provider is down. + if (!isProviderBusy(err) && !isContentFilterRejection(err)) breaker.recordFailure(generation); + else breaker.releaseProbe(generation); + throw err; + } } finally { this.releaseSlot(); } diff --git a/src/state/memory-utils.ts b/src/state/memory-utils.ts index 2a4e3996dd..10d8940bf7 100644 --- a/src/state/memory-utils.ts +++ b/src/state/memory-utils.ts @@ -166,3 +166,5 @@ export function memoryChunkJobs( text: header + "\n\n" + chunk, })); } + +export const MEMORY_WRITE_LOCK = "mem:remember"; diff --git a/test/auto-forget.test.ts b/test/auto-forget.test.ts index f21391d99c..be70d35088 100644 --- a/test/auto-forget.test.ts +++ b/test/auto-forget.test.ts @@ -133,6 +133,31 @@ describe("Auto-Forget Function", () => { expect(older!.isLatest).toBe(false); }); + it.each(["deleted", "demoted", "edited", "retagged"])("keeps the older Memory when the newer one was %s after listing", async (change) => { + const older = makeMemory({ id: "mem_older", createdAt: "2026-01-01T00:00:00Z" }); + const newer = makeMemory({ id: "mem_newer", createdAt: "2026-02-01T00:00:00Z" }); + await kv.set("mem:memories", older.id, older); + await kv.set("mem:memories", newer.id, newer); + const list = kv.list; + kv.list = async (scope: string): Promise => { + const snapshot = await list(scope); + if (scope === "mem:memories") { + if (change === "deleted") await kv.delete(scope, newer.id); + else await kv.set(scope, newer.id, { + ...newer, + ...(change === "demoted" ? { isLatest: false } + : change === "edited" ? { content: "Deploy the unrelated service on Friday" } + : { concepts: ["unrelated"] }), + }); + } + return snapshot; + }; + + expect(await sdk.trigger("mem::auto-forget", {})).toMatchObject({ contradictions: [] }); + expect(await kv.get("mem:memories", older.id)).toMatchObject({ isLatest: true }); + expect(await kv.list("mem:audit")).toEqual([]); + }); + it("removes the older contradicted memory from the live index", async () => { getSearchIndex().clear(); const content = "Use React hooks for state management in all components"; diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index 21668fe8f6..9b0eada0ff 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -115,3 +115,77 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("half-open"); }); }); + +describe("CircuitBreaker half-open probe", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function openThenRecover(): CircuitBreaker { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + return cb; + } + + it("lets one caller probe while half-open and holds the rest", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + expect(cb.isAllowed).toBe(false); + }); + + it("closes after the probe succeeds and lets everyone through", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + cb.recordSuccess(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(true); + }); + + it("reopens with a fresh window after the probe fails", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + cb.recordFailure(); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + }); + + it("does not replace a pending probe after the recovery timeout", () => { + const cb = openThenRecover(); + expect(cb.isAllowed).toBe(true); + expect(cb.isAllowed).toBe(false); + vi.advanceTimersByTime(60_000); + expect(cb.isAllowed).toBe(false); + cb.releaseProbe(); + expect(cb.isAllowed).toBe(true); + }); +}); + +describe("CircuitBreaker probe ownership", () => { + afterEach(() => vi.useRealTimers()); + + it("ignores a stale probe completion after another probe starts", () => { + vi.useFakeTimers(); + const cb = new CircuitBreaker({ failureThreshold: 1 }); + cb.recordFailure(); + vi.advanceTimersByTime(30_000); + const first = cb.acquire()!; + cb.releaseProbe(first); + const second = cb.acquire()!; + cb.recordSuccess(first); + cb.recordFailure(first); + cb.releaseProbe(first); + expect(cb.getState().state).toBe("half-open"); + expect(cb.acquire()).toBeNull(); + cb.recordSuccess(second); + expect(cb.getState().state).toBe("closed"); + }); +}); diff --git a/test/consolidate-dedup.test.ts b/test/consolidate-dedup.test.ts index aa9c689d5d..8fd3b2e3ed 100644 --- a/test/consolidate-dedup.test.ts +++ b/test/consolidate-dedup.test.ts @@ -6,6 +6,9 @@ vi.mock("../src/logger.js", () => ({ import { registerConsolidateFunction } from "../src/functions/consolidate.js"; import { KV } from "../src/state/schema.js"; +import { StateKV } from "../src/state/kv.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../src/state/memory-utils.js"; import { getSearchIndex } from "../src/functions/search.js"; import { memoryToIndexDoc } from "../src/state/memory-utils.js"; import type { CompressedObservation, Memory, MemoryProvider, Session } from "../src/types.js"; @@ -94,6 +97,58 @@ async function setup(concepts: string[][]) { } describe("mem::consolidate dedup", () => { + it.each(["delete", "supersede", "update"])("skips a predecessor changed by %s during the model call", async (change) => { + const backing = makeMockKV(); + const sdk = makeMockSdk(); + const stateSdk = { + trigger: async ({ function_id, payload }: { function_id: string; payload: { scope: string; key: string; value: unknown } }) => { + if (function_id === "state::get") return backing.get(payload.scope, payload.key); + if (function_id === "state::set") return backing.set(payload.scope, payload.key, payload.value); + if (function_id === "state::delete") return backing.delete(payload.scope, payload.key); + if (function_id === "state::list") return backing.list(payload.scope); + throw new Error(`Unexpected function ${function_id}`); + }, + }; + const kv = new StateKV(stateSdk as never); + await kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) await kv.set(KV.observations("s1"), `o${i}`, obs(`o${i}`, ["auth"])); + const first: Memory = { + id: "first", title: "old", content: "old content", type: "pattern", concepts: [], files: [], + sessionIds: ["s1"], strength: 7, version: 1, isLatest: true, project: "api", + createdAt: "2026-01-01", updatedAt: "2026-01-01", sourceObservationIds: ["o0"], + }; + await kv.set(KV.memories, first.id, first); + getSearchIndex().clear(); + getSearchIndex().add(memoryToIndexDoc(first), "memory"); + let release!: (response: string) => void; + let started!: () => void; + const entered = new Promise((resolve) => { started = resolve; }); + const provider = rewordingProvider(); + (provider.compress as ReturnType).mockImplementation(() => { + started(); + return new Promise((resolve) => { release = resolve; }); + }); + registerConsolidateFunction(sdk as never, kv, provider); + const running = sdk.trigger("mem::consolidate", { project: "api", minObservations: 1 }); + await entered; + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + if (change === "delete") { + await kv.delete(KV.memories, first.id); + getSearchIndex().remove(first.id); + } else if (change === "supersede") { + await kv.set(KV.memories, first.id, { ...first, isLatest: false }); + } else { + await kv.set(KV.memories, first.id, { ...first, updatedAt: "2026-02-01", content: "updated" }); + } + }); + release("patternnewnew content"); + expect(await running).toMatchObject({ consolidated: 0 }); + const memories = await kv.list(KV.memories); + expect(memories).toHaveLength(change === "delete" ? 0 : 1); + if (change === "supersede") expect(memories[0].isLatest).toBe(false); + if (change === "update") expect(memories[0].content).toBe("updated"); + if (change === "delete") expect(getSearchIndex().has(first.id)).toBe(false); + }); it("writes nothing on a second run over unchanged input", async () => { const { run, kv } = await setup([["auth"], ["auth"], ["auth"]]); expect(await run()).toMatchObject({ consolidated: 1 }); diff --git a/test/evict.test.ts b/test/evict.test.ts index b4a8b43364..53f241e22c 100644 --- a/test/evict.test.ts +++ b/test/evict.test.ts @@ -708,7 +708,7 @@ describe("mem::evict default importance floor", () => { .sort(); const keptMem = async () => (await kv.list(KV.memories)).map((m) => m.id).sort(); - return { run, keptObs, keptMem }; + return { run, keptObs, keptMem, kv }; } it("removes a one-day-old Observation of importance 4 and keeps importance 5 and a fresh one", async () => { @@ -737,6 +737,25 @@ describe("mem::evict default importance floor", () => { expect(await t.keptMem()).toEqual(["latest", "young"]); }); + it.each([ + ["expiredMemories", "expired_memory", { ...memory("old", 100, true), forgetAfter: daysAgo(1) }], + ["nonLatestMemories", "old_non_latest_memory", memory("old", 100, false)], + ] as const)("counts and audits %s when image release fails", async (counter, reason, old) => { + const t = setup([], [{ ...old, imageRef: "images/old.png" }]); + const get = t.kv.get; + vi.spyOn(t.kv, "get").mockImplementation((scope, key) => { + if (scope === KV.imageRefs) throw new Error("image refs unavailable"); + return get(scope, key); + }); + + expect(await t.run()).toMatchObject({ [counter]: 1 }); + expect(await t.keptMem()).toEqual([]); + expect(await t.kv.list(KV.audit)).toContainEqual(expect.objectContaining({ + operation: "delete", + details: expect.objectContaining({ reason }), + })); + }); + it("reports the default removals on a dry run without deleting", async () => { const t = setup( [obs("ses_d", "low", 1, 4), obs("ses_d", "ok", 1, 5)], diff --git a/test/memory-write-locks.test.ts b/test/memory-write-locks.test.ts new file mode 100644 index 0000000000..17329175c8 --- /dev/null +++ b/test/memory-write-locks.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerRememberFunction } from "../src/functions/remember.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerGovernanceFunction } from "../src/functions/governance.js"; +import { registerAutoForgetFunction } from "../src/functions/auto-forget.js"; +import { registerEvictFunction } from "../src/functions/evict.js"; +import { registerRetentionFunctions } from "../src/functions/retention.js"; +import type { Memory } from "../src/types.js"; + +const MEMORIES = "mem:memories"; +const tick = () => new Promise((r) => setTimeout(r, 15)); + +function gatedKV() { + const store = new Map>(); + let gate: { scope: string; key: string; reached: () => void; open: Promise } | null = null; + const clone = (v: T): T => structuredClone(v); + const kv = { + get: async (scope: string, key: string): Promise => { + const row = store.get(scope)?.get(key); + return row === undefined ? null : clone(row as T); + }, + set: async (scope: string, key: string, data: T): Promise => { + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + g.reached(); + await g.open; + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, clone(data)); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []).map((v) => clone(v as T)), + }; + function pauseNextSet(scope: string, key: string) { + let release!: () => void; + let reached!: () => void; + const open = new Promise((r) => (release = r)); + const hit = new Promise((r) => (reached = r)); + gate = { scope, key, reached, open }; + return { release, hit }; + } + return { kv, pauseNextSet }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + functions.set(typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, handler); + }, + registerTrigger: () => {}, + trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +function makeMemory(overrides: Partial = {}): Memory { + return { + id: "mem_1", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + type: "fact", + title: "Lock probe", + content: "zebra quartz lattice meridian harbor", + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: true, + ...overrides, + }; +} + +describe("Memory writers serialize with deletes and relates", () => { + let sdk: ReturnType; + let env: ReturnType; + + beforeEach(() => { + sdk = mockSdk(); + env = gatedKV(); + registerRememberFunction(sdk as never, env.kv as never); + registerRelationsFunction(sdk as never, env.kv as never); + registerGovernanceFunction(sdk as never, env.kv as never); + registerAutoForgetFunction(sdk as never, env.kv as never); + registerEvictFunction(sdk as never, env.kv as never); + registerRetentionFunctions(sdk as never, env.kv as never); + }); + + async function deleteDuringReinforce( + seed: Partial, + del: () => Promise, + ) { + const x = makeMemory({ id: "mem_x", ...seed }); + await env.kv.set(MEMORIES, x.id, x); + const gate = env.pauseNextSet(MEMORIES, x.id); + const save = sdk.trigger("mem::remember", { content: x.content }); + await gate.hit; + const deletion = del(); + await tick(); + gate.release(); + await Promise.all([save, deletion]); + return env.kv.get(MEMORIES, x.id); + } + + it("governance-delete is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::governance-delete", { memoryIds: ["mem_x"] }), + ); + expect(left).toBeNull(); + }); + + it("governance-bulk is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::governance-bulk", { type: ["fact"] }), + ); + expect(left).toBeNull(); + }); + + it("forget is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::forget", { memoryId: "mem_x" }), + ); + expect(left).toBeNull(); + }); + + it("auto-forget TTL is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({ forgetAfter: "2020-01-01T00:00:00Z" }, () => + sdk.trigger("mem::auto-forget", {}), + ); + expect(left).toBeNull(); + }); + + it("evict TTL is not undone by a remember that read the Memory first", async () => { + const left = await deleteDuringReinforce({ forgetAfter: "2020-01-01T00:00:00Z" }, () => + sdk.trigger("mem::evict", {}), + ); + expect(left).toBeNull(); + }); + + it("retention-evict is not undone by a remember that read the Memory first", async () => { + await env.kv.set("mem:retention", "mem_x", { + memoryId: "mem_x", + score: 0, + source: "episodic", + }); + const left = await deleteDuringReinforce({}, () => + sdk.trigger("mem::retention-evict", { threshold: 1 }), + ); + expect(left).toBeNull(); + }); + + it("two relates sharing a Memory keep both relatedIds", async () => { + for (const id of ["mem_a", "mem_b", "mem_c"]) { + await env.kv.set(MEMORIES, id, makeMemory({ id, content: `${id} unrelated text` })); + } + await Promise.all([ + sdk.trigger("mem::relate", { sourceId: "mem_a", targetId: "mem_b", type: "related" }), + sdk.trigger("mem::relate", { sourceId: "mem_a", targetId: "mem_c", type: "related" }), + ]); + const a = await env.kv.get(MEMORIES, "mem_a"); + expect([...(a!.relatedIds ?? [])].sort()).toEqual(["mem_b", "mem_c"]); + }); + + it("a relate that read a Memory before evolve does not mark the superseded version latest", async () => { + await env.kv.set(MEMORIES, "mem_a", makeMemory({ id: "mem_a", content: "alpha record" })); + await env.kv.set(MEMORIES, "mem_b", makeMemory({ id: "mem_b", content: "beta record" })); + const gate = env.pauseNextSet(MEMORIES, "mem_a"); + const relate = sdk.trigger("mem::relate", { + sourceId: "mem_a", + targetId: "mem_b", + type: "related", + }); + await gate.hit; + const evolve = sdk.trigger("mem::evolve", { memoryId: "mem_a", newContent: "alpha record v2" }); + await tick(); + gate.release(); + await Promise.all([relate, evolve]); + const a = await env.kv.get(MEMORIES, "mem_a"); + expect(a!.isLatest).toBe(false); + }); +}); diff --git a/test/resilient-half-open.test.ts b/test/resilient-half-open.test.ts new file mode 100644 index 0000000000..f9061c43bb --- /dev/null +++ b/test/resilient-half-open.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +describe("ResilientProvider half-open breaker", () => { + afterEach(() => vi.useRealTimers()); + + it("sends one probe to a provider that is still down, not every queued call", async () => { + vi.useFakeTimers(); + let calls = 0; + const inner: MemoryProvider = { + name: "down", + compress: async () => { + calls++; + throw new Error("connection refused"); + }, + summarize: async () => "", + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + expect(calls).toBe(3); + + vi.advanceTimersByTime(30_000); + calls = 0; + await Promise.allSettled( + Array.from({ length: 8 }, () => provider.compress("s", "u")), + ); + expect(calls).toBe(1); + }); + it.each([ + ["busy", Object.assign(new Error("busy"), { status: 429 })], + ["filtered", new Error("content_filter")], + ])("releases a %s probe for the next call", async (_name, rejection) => { + vi.useFakeTimers(); + let mode = "down"; + const inner: MemoryProvider = { + name: "synthetic", + compress: async () => { + if (mode === "down") throw new Error("down"); + if (mode === "reject") throw rejection; + return "ok"; + }, + summarize: async () => "", + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + vi.advanceTimersByTime(30_000); + mode = "reject"; + await expect(provider.compress("s", "u")).rejects.toThrow(); + mode = "ok"; + await expect(provider.compress("s", "u")).resolves.toBe("ok"); + }); + + it("rejects an open circuit without waiting for a shared slot", async () => { + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + const inner: MemoryProvider = { + name: "synthetic", + compress: async () => { throw new Error("down"); }, + summarize: async () => { await pending; return "ok"; }, + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) await provider.compress("s", "u").catch(() => {}); + const slots = Array.from({ length: 4 }, () => provider.summarize("s", "u")); + await Promise.resolve(); + try { + await expect(provider.compress("s", "u")).rejects.toThrow("circuit_breaker_open"); + } finally { + release(); + await Promise.all(slots); + } + }); + +});