diff --git a/src/functions/cascade.ts b/src/functions/cascade.ts index 2167c506d5..b29596764b 100644 --- a/src/functions/cascade.ts +++ b/src/functions/cascade.ts @@ -1,6 +1,7 @@ import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import type { Memory, GraphNode, GraphEdge } from "../types.js"; import { recordAudit } from "./audit.js"; @@ -28,10 +29,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void { for (const node of nodes) { if (node.stale) continue; const overlap = (node.sourceObservationIds ?? []).some((id) => obsIds.has(id)); - if (overlap) { - node.stale = true; - node.updatedAt = now; - await kv.set(KV.graphNodes, node.id, node); + if (!overlap) continue; + const flagged = await withKeyedLock("graph:persist", async () => { + const latest = (await kv.get(KV.graphNodes, node.id)) ?? node; + if (latest.stale) return false; + await kv.set(KV.graphNodes, node.id, { ...latest, stale: true, updatedAt: now }); + return true; + }); + if (flagged) { await recordAudit(kv, "consolidate", "mem::cascade-update", [node.id], { resourceType: "GraphNode", change: "marked stale from superseded memory", @@ -45,9 +50,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void { for (const edge of edges) { if (edge.stale) continue; const overlap = (edge.sourceObservationIds ?? []).some((id) => obsIds.has(id)); - if (overlap) { - edge.stale = true; - await kv.set(KV.graphEdges, edge.id, edge); + if (!overlap) continue; + const flagged = await withKeyedLock("graph:persist", async () => { + const latest = (await kv.get(KV.graphEdges, edge.id)) ?? edge; + if (latest.stale) return false; + await kv.set(KV.graphEdges, edge.id, { ...latest, stale: true }); + return true; + }); + if (flagged) { await recordAudit(kv, "consolidate", "mem::cascade-update", [edge.id], { resourceType: "GraphEdge", change: "marked stale from superseded memory", diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 6d0128f36a..747cdd5307 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -29,6 +29,7 @@ import { normalizeAccessLog } from "./access-tracker.js"; import { KV } from "../state/schema.js"; import { checkPayloadFrameSize } from "../state/frame-guard.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { VERSION } from "../version.js"; import { recordAudit } from "./audit.js"; import { indexRecords } from "./search.js"; @@ -372,13 +373,17 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void await kv.list(KV.insights).catch(() => []), (i) => kv.delete(KV.insights, i.id), ); - await runChunked( - await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), - (n) => kv.delete(KV.graphNodes, n.id), + await withKeyedLock("graph:persist", async () => + runChunked( + await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), + (n) => kv.delete(KV.graphNodes, n.id), + ), ); - await runChunked( - await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), - (e) => kv.delete(KV.graphEdges, e.id), + await withKeyedLock("graph:persist", async () => + runChunked( + await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), + (e) => kv.delete(KV.graphEdges, e.id), + ), ); await runChunked( await kv.list<{ id: string }>(KV.semantic).catch(() => []), @@ -407,17 +412,19 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void const indexMems: Memory[] = []; await runChunked(importData.sessions, async (session) => { - if (strategy === "skip") { - const existing = await kv - .get(KV.sessions, session.id) - .catch(() => null); - if (existing) { - stats.skipped++; - return; + await withKeyedLock(`obs:${session.id}`, async () => { + if (strategy === "skip") { + const existing = await kv + .get(KV.sessions, session.id) + .catch(() => null); + if (existing) { + stats.skipped++; + return; + } } - } - await kv.set(KV.sessions, session.id, session); - stats.sessions++; + await kv.set(KV.sessions, session.id, session); + stats.sessions++; + }); }); for (const [sessionId, obs] of Object.entries(importData.observations)) { @@ -473,22 +480,26 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void }); if (importData.graphNodes) { - await runChunked(importData.graphNodes, async (node) => { - if (strategy === "skip") { - const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); - if (existing) { stats.skipped++; return; } - } - await kv.set(KV.graphNodes, node.id, node); - }); + await withKeyedLock("graph:persist", () => + runChunked(importData.graphNodes!, async (node) => { + if (strategy === "skip") { + const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); + if (existing) { stats.skipped++; return; } + } + await kv.set(KV.graphNodes, node.id, node); + }), + ); } if (importData.graphEdges) { - await runChunked(importData.graphEdges, async (edge) => { - if (strategy === "skip") { - const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); - if (existing) { stats.skipped++; return; } - } - await kv.set(KV.graphEdges, edge.id, edge); - }); + await withKeyedLock("graph:persist", () => + runChunked(importData.graphEdges!, async (edge) => { + if (strategy === "skip") { + const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); + if (existing) { stats.skipped++; return; } + } + await kv.set(KV.graphEdges, edge.id, edge); + }), + ); } if (importData.semanticMemories) { await runChunked(importData.semanticMemories, async (sem) => { diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 62e7343e3b..081887318a 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -9,6 +9,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { GRAPH_EXTRACTION_SYSTEM, buildGraphExtractionPrompt, @@ -554,129 +555,131 @@ export async function persistGraphDelta( edges: GraphEdge[], obsIds: string[], ): Promise<{ newNodeCount: number; newEdgeCount: number }> { - const snap = (await readSnapshot(kv)) ?? emptySnapshot(); - const capturedAt = new Date().toISOString(); - let newNodeCount = 0; - let newEdgeCount = 0; - // Merge-only batches mutate cached topNodes/topEdges entries without - // changing the counts; track that separately so the snapshot still persists. - let snapMutated = false; - const newEdgesForTopCheck: GraphEdge[] = []; - // When a freshly-minted node merges into an existing row via the name - // index, edges in the same batch still reference the fresh id. Remap edge - // endpoints to the persisted ids so edges never dangle and re-runs hit the - // same edge-index key instead of duplicating. - const idRemap = new Map(); - - for (const node of nodes) { - const indexKey = nameIndexKey(node.type, node.name); - const existingId = await kv.get(KV.graphNameIndex, indexKey); - - let existing: GraphNode | null = null; - if (existingId) { - existing = await kv.get(KV.graphNodes, existingId); - // #825 follow-up: name-index lookups can resolve into - // pre-reset rows. Drop them so extract writes a fresh - // node + index entry instead of silently reconnecting - // to a legacy orphan (which would keep the snapshot at - // 0 forever after a reset). - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; + return withKeyedLock("graph:persist", async () => { + const snap = (await readSnapshot(kv)) ?? emptySnapshot(); + const capturedAt = new Date().toISOString(); + let newNodeCount = 0; + let newEdgeCount = 0; + // Merge-only batches mutate cached topNodes/topEdges entries without + // changing the counts; track that separately so the snapshot still persists. + let snapMutated = false; + const newEdgesForTopCheck: GraphEdge[] = []; + // When a freshly-minted node merges into an existing row via the name + // index, edges in the same batch still reference the fresh id. Remap edge + // endpoints to the persisted ids so edges never dangle and re-runs hit the + // same edge-index key instead of duplicating. + const idRemap = new Map(); + + for (const node of nodes) { + const indexKey = nameIndexKey(node.type, node.name); + const existingId = await kv.get(KV.graphNameIndex, indexKey); + + let existing: GraphNode | null = null; + if (existingId) { + existing = await kv.get(KV.graphNodes, existingId); + // #825 follow-up: name-index lookups can resolve into + // pre-reset rows. Drop them so extract writes a fresh + // node + index entry instead of silently reconnecting + // to a legacy orphan (which would keep the snapshot at + // 0 forever after a reset). + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } } - } - if (existing) { - idRemap.set(node.id, existing.id); - const merged = mergeNode(existing, node, obsIds, capturedAt); - await kv.set(KV.graphNodes, existing.id, merged); - // Update topNodes entry if present so a stale clone isn't - // returned from the snapshot fast path. - const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id); - if (topIdx !== -1) { - snap.topNodes[topIdx] = merged; - snapMutated = true; - } - } else { - await kv.set(KV.graphNodes, node.id, node); - await kv.set(KV.graphNameIndex, indexKey, node.id); - await kv.set(KV.graphNodeDegree, node.id, 0); - snap.stats.totalNodes += 1; - snap.stats.nodesByType[node.type] = - (snap.stats.nodesByType[node.type] ?? 0) + 1; - newNodeCount += 1; - if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { - // Degree 0 still beats an empty slot — sit at the tail - // until edges arrive and promote. - snap.topNodes.push(node); - snap.topDegrees[node.id] = 0; + if (existing) { + idRemap.set(node.id, existing.id); + const merged = mergeNode(existing, node, obsIds, capturedAt); + await kv.set(KV.graphNodes, existing.id, merged); + // Update topNodes entry if present so a stale clone isn't + // returned from the snapshot fast path. + const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id); + if (topIdx !== -1) { + snap.topNodes[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphNodes, node.id, node); + await kv.set(KV.graphNameIndex, indexKey, node.id); + await kv.set(KV.graphNodeDegree, node.id, 0); + snap.stats.totalNodes += 1; + snap.stats.nodesByType[node.type] = + (snap.stats.nodesByType[node.type] ?? 0) + 1; + newNodeCount += 1; + if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { + // Degree 0 still beats an empty slot — sit at the tail + // until edges arrive and promote. + snap.topNodes.push(node); + snap.topDegrees[node.id] = 0; + } } } - } - for (const rawEdge of edges) { - const edge: GraphEdge = { - ...rawEdge, - sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId, - targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId, - }; - const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type); - const existingId = await kv.get(KV.graphEdgeKey, eKey); - - let existing: GraphEdge | null = null; - if (existingId) { - existing = await kv.get(KV.graphEdges, existingId); - // Same #825 orphan check as the node path above. - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; + for (const rawEdge of edges) { + const edge: GraphEdge = { + ...rawEdge, + sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId, + targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId, + }; + const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type); + const existingId = await kv.get(KV.graphEdgeKey, eKey); + + let existing: GraphEdge | null = null; + if (existingId) { + existing = await kv.get(KV.graphEdges, existingId); + // Same #825 orphan check as the node path above. + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } } - } - if (existing) { - const merged = mergeEdge(existing, obsIds); - await kv.set(KV.graphEdges, existing.id, merged); - // Replace cached topEdges entry too if present. - const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); - if (topIdx !== -1) { - snap.topEdges[topIdx] = merged; - snapMutated = true; + if (existing) { + const merged = mergeEdge(existing, obsIds); + await kv.set(KV.graphEdges, existing.id, merged); + // Replace cached topEdges entry too if present. + const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); + if (topIdx !== -1) { + snap.topEdges[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphEdges, edge.id, edge); + await kv.set(KV.graphEdgeKey, eKey, edge.id); + snap.stats.totalEdges += 1; + snap.stats.edgesByType[edge.type] = + (snap.stats.edgesByType[edge.type] ?? 0) + 1; + newEdgeCount += 1; + await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); + await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); + newEdgesForTopCheck.push(edge); } - } else { - await kv.set(KV.graphEdges, edge.id, edge); - await kv.set(KV.graphEdgeKey, eKey, edge.id); - snap.stats.totalEdges += 1; - snap.stats.edgesByType[edge.type] = - (snap.stats.edgesByType[edge.type] ?? 0) + 1; - newEdgeCount += 1; - await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); - await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); - newEdgesForTopCheck.push(edge); } - } - // Push newly-added edges into snapshot.topEdges if both - // endpoints are in the top-N (post-degree-delta). Done after - // all degree updates so the topIds set is stable. - for (const edge of newEdgesForTopCheck) { - snapshotPushEdgeIfBothInTop(snap, edge); - } + // Push newly-added edges into snapshot.topEdges if both + // endpoints are in the top-N (post-degree-delta). Done after + // all degree updates so the topIds set is stable. + for (const edge of newEdgesForTopCheck) { + snapshotPushEdgeIfBothInTop(snap, edge); + } - if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) { - snap.updatedAt = capturedAt; - snap.dirty = false; - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); - } + if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) { + snap.updatedAt = capturedAt; + snap.dirty = false; + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); + } - return { newNodeCount, newEdgeCount }; + return { newNodeCount, newEdgeCount }; + }); } export function registerGraphFunction( @@ -1014,70 +1017,74 @@ export function registerGraphFunction( } try { - const [nodes, edges] = await withTimeout( - Promise.all([ - kv.list(KV.graphNodes), - kv.list(KV.graphEdges), - ]), - LIVE_ENUMERATION_BUDGET_MS, - "graph-snapshot-rebuild enumeration", - ); + const outcome = await withKeyedLock("graph:persist", async () => { + const [nodes, edges] = await withTimeout( + Promise.all([ + kv.list(KV.graphNodes), + kv.list(KV.graphEdges), + ]), + LIVE_ENUMERATION_BUDGET_MS, + "graph-snapshot-rebuild enumeration", + ); + + if (nodes.length > REBUILD_SAFE_NODE_CEILING) { + return { tooLargeNodeCount: nodes.length }; + } + + const liveNodes = nodes.filter((n) => !n.stale); + const liveEdges = edges.filter((e) => !e.stale); + const degree = new Map(); + for (const e of liveEdges) { + degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1); + degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1); + } + const BATCH_SIZE = 100; + for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) { + const batch = liveNodes.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.flatMap((n) => [ + kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id), + kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0), + ]), + ); + } + for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) { + const batch = liveEdges.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map((e) => + kv.set( + KV.graphEdgeKey, + edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type), + e.id, + ), + ), + ); + } - if (nodes.length > REBUILD_SAFE_NODE_CEILING) { + const rebuilt = buildSnapshotFromArrays(nodes, edges); + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, rebuilt); + return { snap: rebuilt }; + }); + + if ("tooLargeNodeCount" in outcome) { + const totalNodes = outcome.tooLargeNodeCount; logger.warn("Graph snapshot rebuild aborted: corpus too large", { - totalNodes: nodes.length, + totalNodes, ceiling: REBUILD_SAFE_NODE_CEILING, }); return { success: false, tooLarge: true, - totalNodes: nodes.length, + totalNodes, ceiling: REBUILD_SAFE_NODE_CEILING, error: - `Corpus has ${nodes.length} graph nodes; safe-rebuild ceiling ` + + `Corpus has ${totalNodes} graph nodes; safe-rebuild ceiling ` + `is ${REBUILD_SAFE_NODE_CEILING}. Run POST /agentmemory/graph/reset ` + `to wipe and let future extracts rebuild incrementally.`, }; } - // Backfill the targeted-lookup indexes so post-rebuild - // graph-extract calls hit the O(1) path instead of falling - // through to the (already-removed) full-scope scan. Batch - // writes via Promise.all to avoid N sequential round-trips — - // BATCH_SIZE bounds in-flight writes so we don't open thousands - // of concurrent state channels on huge corpora. - const liveNodes = nodes.filter((n) => !n.stale); - const liveEdges = edges.filter((e) => !e.stale); - const degree = new Map(); - for (const e of liveEdges) { - degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1); - degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1); - } - const BATCH_SIZE = 100; - for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) { - const batch = liveNodes.slice(i, i + BATCH_SIZE); - await Promise.all( - batch.flatMap((n) => [ - kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id), - kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0), - ]), - ); - } - for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) { - const batch = liveEdges.slice(i, i + BATCH_SIZE); - await Promise.all( - batch.map((e) => - kv.set( - KV.graphEdgeKey, - edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type), - e.id, - ), - ), - ); - } - - const snap = buildSnapshotFromArrays(nodes, edges); - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); + const { snap } = outcome; const tookMs = Date.now() - started; logger.info("Graph snapshot rebuilt", { totalNodes: snap.stats.totalNodes, @@ -1134,7 +1141,9 @@ export function registerGraphFunction( ...emptySnapshot(), resetAt: new Date().toISOString(), }; - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot); + await withKeyedLock("graph:persist", () => + kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot), + ); const counts: Record = { [KV.graphSnapshot]: 1, }; diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts index 139765adf2..0dd7e95753 100644 --- a/src/functions/mesh.ts +++ b/src/functions/mesh.ts @@ -119,7 +119,7 @@ async function lwwMergeGraphNodes( if (!item.id || typeof item.id !== "string") continue; const ts = graphNodeTs(item); if (!ts || Number.isNaN(new Date(ts).getTime())) continue; - const wrote = await withKeyedLock(`mem:gnode:${item.id}`, async () => { + const wrote = await withKeyedLock("graph:persist", async () => { const existing = await kv.get(KV.graphNodes, item.id); if (!existing) { await kv.set(KV.graphNodes, item.id, item); @@ -136,6 +136,33 @@ async function lwwMergeGraphNodes( return count; } +async function lwwMergeGraphEdges( + kv: StateKV, + items: GraphEdge[] | undefined, +): Promise { + if (!items || !Array.isArray(items)) return 0; + let count = 0; + for (const item of items) { + if (!item.id || typeof item.id !== "string") continue; + const ts = item.createdAt; + if (!ts || Number.isNaN(new Date(ts).getTime())) continue; + const wrote = await withKeyedLock("graph:persist", async () => { + const existing = await kv.get(KV.graphEdges, item.id); + if (!existing) { + await kv.set(KV.graphEdges, item.id, item); + return true; + } + if (new Date(ts) > new Date(existing.createdAt)) { + await kv.set(KV.graphEdges, item.id, item); + return true; + } + return false; + }); + if (wrote) count++; + } + return count; +} + export function registerMeshFunction( sdk: IIIClient, kv: StateKV, @@ -361,7 +388,7 @@ export function registerMeshFunction( } } accepted += await lwwMergeGraphNodes(kv, data.graphNodes); - accepted += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt"); + accepted += await lwwMergeGraphEdges(kv, data.graphEdges); await recordAudit(kv, "mesh_sync", "mem::mesh-receive", [], { action: "mesh.receive", accepted, @@ -488,7 +515,7 @@ async function applySyncData( applied += await lwwMergeGraphNodes(kv, data.graphNodes); } if (scopes.includes("graph:edges")) { - applied += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt"); + applied += await lwwMergeGraphEdges(kv, data.graphEdges); } return applied; diff --git a/src/functions/migrate.ts b/src/functions/migrate.ts index 8cabef5315..350a533250 100644 --- a/src/functions/migrate.ts +++ b/src/functions/migrate.ts @@ -3,6 +3,7 @@ import { resolve } from "node:path"; import { homedir } from "node:os"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import type { Memory, Session, @@ -150,7 +151,9 @@ export function registerMigrateFunction(sdk: IIIClient, kv: StateKV): void { status: "completed", observationCount: 0, }; - await kv.set(KV.sessions, session.id, session); + await withKeyedLock(`obs:${session.id}`, () => + kv.set(KV.sessions, session.id, session), + ); sessionCount++; } diff --git a/src/functions/replay.ts b/src/functions/replay.ts index aea735ff98..d158b91384 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -12,6 +12,7 @@ import type { import { importOrigin } from "../types.js"; import type { StateKV } from "../state/kv.js"; import { KV, generateId, fingerprintId } from "../state/schema.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { parseJsonlText } from "../replay/jsonl-parser.js"; import { resetLessonIndex } from "./lessons.js"; import { projectTimeline, type Timeline } from "../replay/timeline.js"; @@ -394,46 +395,48 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void { ? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) : undefined; - const existing = await kv.get(KV.sessions, parsed.sessionId); - if (existing) { - existing.observationCount = - (existing.observationCount || 0) + parsed.observations.length; - if (parsed.endedAt > (existing.endedAt || "")) { - existing.endedAt = parsed.endedAt; + await withKeyedLock(`obs:${parsed.sessionId}`, async () => { + const existing = await kv.get(KV.sessions, parsed.sessionId); + if (existing) { + existing.observationCount = + (existing.observationCount || 0) + parsed.observations.length; + if (parsed.endedAt > (existing.endedAt || "")) { + existing.endedAt = parsed.endedAt; + } + if (existing.status === "active") existing.status = "completed"; + const existingTags = existing.tags || []; + if (!existingTags.includes("jsonl-import")) { + existing.tags = [...existingTags, "jsonl-import"]; + } + if (!existing.firstPrompt && firstPrompt) { + existing.firstPrompt = firstPrompt; + } + // #775: re-key on parsed.sessionId, not existing.id. Older + // session rows may be missing the `id` field; existing.id + // would then be undefined, JSON.stringify would drop the + // `key` from the state::set payload, and the engine would + // reject the call with `missing field \`key\``. Because the + // rejection aborts the whole import handler, a single + // legacy row killed the entire batch. parsed.sessionId is + // always populated (parseJsonlText has a three-level + // fallback) and is what we just used to read the row. + if (!existing.id) existing.id = parsed.sessionId; + await kv.set(KV.sessions, parsed.sessionId, existing); + } else { + const session: Session = { + id: parsed.sessionId, + project: parsed.project, + cwd: parsed.cwd, + startedAt: parsed.startedAt, + endedAt: parsed.endedAt, + status: "completed", + observationCount: parsed.observations.length, + tags: ["jsonl-import"], + firstPrompt, + }; + await kv.set(KV.sessions, session.id, session); } - if (existing.status === "active") existing.status = "completed"; - const existingTags = existing.tags || []; - if (!existingTags.includes("jsonl-import")) { - existing.tags = [...existingTags, "jsonl-import"]; - } - if (!existing.firstPrompt && firstPrompt) { - existing.firstPrompt = firstPrompt; - } - // #775: re-key on parsed.sessionId, not existing.id. Older - // session rows may be missing the `id` field; existing.id - // would then be undefined, JSON.stringify would drop the - // `key` from the state::set payload, and the engine would - // reject the call with `missing field \`key\``. Because the - // rejection aborts the whole import handler, a single - // legacy row killed the entire batch. parsed.sessionId is - // always populated (parseJsonlText has a three-level - // fallback) and is what we just used to read the row. - if (!existing.id) existing.id = parsed.sessionId; - await kv.set(KV.sessions, parsed.sessionId, existing); - } else { - const session: Session = { - id: parsed.sessionId, - project: parsed.project, - cwd: parsed.cwd, - startedAt: parsed.startedAt, - endedAt: parsed.endedAt, - status: "completed", - observationCount: parsed.observations.length, - tags: ["jsonl-import"], - firstPrompt, - }; - await kv.set(KV.sessions, session.id, session); - } + }); const compressed: CompressedObservation[] = []; await Promise.all( diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 483240e14d..066738451c 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -12,6 +12,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; import { VERSION } from "../version.js"; import { logger } from "../logger.js"; @@ -195,7 +196,9 @@ export function registerSnapshotFunction( if (state.sessions) { for (const session of state.sessions) { - await kv.set(KV.sessions, session.id, session); + await withKeyedLock(`obs:${session.id}`, () => + kv.set(KV.sessions, session.id, session), + ); } } if (state.memories) { @@ -205,7 +208,9 @@ export function registerSnapshotFunction( } if (state.graphNodes) { for (const node of state.graphNodes) { - await kv.set(KV.graphNodes, node.id, node); + await withKeyedLock("graph:persist", () => + kv.set(KV.graphNodes, node.id, node), + ); } } if (state.observations) { diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index aed6730b5f..18d13c1fe5 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -9,6 +9,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; const TEMPORAL_EXTRACTION_SYSTEM = `You are a temporal knowledge extraction engine. Given observations, extract entities AND their temporal relationships with full context metadata. @@ -186,76 +187,78 @@ export function registerTemporalGraphFunctions( const obsIds = data.observations.map((o) => o.id); const { nodes, edges } = parseTemporalGraphXml(response, obsIds); - const existingNodes = await kv.list(KV.graphNodes); - const existingEdges = await kv.list(KV.graphEdges); - - const idRemap = new Map(); - for (const node of nodes) { - const existing = existingNodes.find( - (n) => - n.name === node.name && n.type === node.type, - ); - if (existing) { - const oldId = node.id; - const merged = { - ...existing, - sourceObservationIds: [ - ...new Set([ - ...existing.sourceObservationIds, - ...obsIds, - ]), - ], - properties: { ...existing.properties, ...node.properties }, - updatedAt: new Date().toISOString(), - aliases: [ - ...new Set([ - ...(existing.aliases || []), - ...(node.aliases || []), - ]), - ], - }; - if (merged.aliases.length === 0) delete (merged as any).aliases; - await kv.set(KV.graphNodes, existing.id, merged); - node.id = existing.id; - idRemap.set(oldId, existing.id); - } else { - await kv.set(KV.graphNodes, node.id, node); - existingNodes.push(node); + await withKeyedLock("graph:persist", async () => { + const existingNodes = await kv.list(KV.graphNodes); + const existingEdges = await kv.list(KV.graphEdges); + + const idRemap = new Map(); + for (const node of nodes) { + const existing = existingNodes.find( + (n) => + n.name === node.name && n.type === node.type, + ); + if (existing) { + const oldId = node.id; + const merged = { + ...existing, + sourceObservationIds: [ + ...new Set([ + ...existing.sourceObservationIds, + ...obsIds, + ]), + ], + properties: { ...existing.properties, ...node.properties }, + updatedAt: new Date().toISOString(), + aliases: [ + ...new Set([ + ...(existing.aliases || []), + ...(node.aliases || []), + ]), + ], + }; + if (merged.aliases.length === 0) delete (merged as any).aliases; + await kv.set(KV.graphNodes, existing.id, merged); + node.id = existing.id; + idRemap.set(oldId, existing.id); + } else { + await kv.set(KV.graphNodes, node.id, node); + existingNodes.push(node); + } } - } - for (const edge of edges) { - if (idRemap.has(edge.sourceNodeId)) { - edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!; + for (const edge of edges) { + if (idRemap.has(edge.sourceNodeId)) { + edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!; + } + if (idRemap.has(edge.targetNodeId)) { + edge.targetNodeId = idRemap.get(edge.targetNodeId)!; + } + const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`; + const existingEdge = existingEdges.find( + (e) => + `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` === + existingKey, + ); + + if (existingEdge) { + const updatedOld = { + ...existingEdge, + isLatest: false, + tvalidEnd: + existingEdge.tvalidEnd || new Date().toISOString(), + supersededBy: edge.id, + }; + await kv.set(KV.graphEdges, existingEdge.id, updatedOld); + + await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld); + + edge.version = (existingEdge.version || 1) + 1; + } + + await kv.set(KV.graphEdges, edge.id, edge); + existingEdges.push(edge); } - if (idRemap.has(edge.targetNodeId)) { - edge.targetNodeId = idRemap.get(edge.targetNodeId)!; - } - const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`; - const existingEdge = existingEdges.find( - (e) => - `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` === - existingKey, - ); - - if (existingEdge) { - const updatedOld = { - ...existingEdge, - isLatest: false, - tvalidEnd: - existingEdge.tvalidEnd || new Date().toISOString(), - supersededBy: edge.id, - }; - await kv.set(KV.graphEdges, existingEdge.id, updatedOld); - - await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld); - - edge.version = (existingEdge.version || 1) + 1; - } - - await kv.set(KV.graphEdges, edge.id, edge); - existingEdges.push(edge); - } + }); logger.info("Temporal graph extraction complete", { nodes: nodes.length, diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d8a447f2e0..54f5a0e857 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -721,7 +721,7 @@ export function registerApiTriggers( ? body.agentId.trim().slice(0, 128) : undefined; const agentId = requestAgentId ?? getAgentId(); - const session: Session = { + const freshSession: Session = { id: sessionId, project, cwd, @@ -732,7 +732,18 @@ export function registerApiTriggers( ...(title ? { firstPrompt: title.slice(0, 200) } : {}), ...(agentId ? { agentId } : {}), }; - await kv.set(KV.sessions, sessionId, session); + const session = await withKeyedLock(`obs:${sessionId}`, async () => { + const existing = await kv.get(KV.sessions, sessionId); + const merged: Session = { + ...freshSession, + observationCount: existing?.observationCount ?? freshSession.observationCount, + firstPrompt: freshSession.firstPrompt ?? existing?.firstPrompt, + summary: freshSession.summary ?? existing?.summary, + commitShas: existing?.commitShas ?? freshSession.commitShas, + }; + await kv.set(KV.sessions, sessionId, merged); + return merged; + }); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } @@ -765,10 +776,12 @@ export function registerApiTriggers( body: { error: "sessionId is required and must be a non-empty string" }, }; } - await kv.update(KV.sessions, sessionId, [ - { type: "set", path: "endedAt", value: new Date().toISOString() }, - { type: "set", path: "status", value: "completed" }, - ]); + await withKeyedLock(`obs:${sessionId}`, () => + kv.update(KV.sessions, sessionId, [ + { type: "set", path: "endedAt", value: new Date().toISOString() }, + { type: "set", path: "status", value: "completed" }, + ]), + ); // Fan out session-stopped lifecycle (non-blocking). try { sdk.trigger({ @@ -861,7 +874,7 @@ export function registerApiTriggers( }); if (sessionId) { - await withKeyedLock(`session:${sessionId}`, async () => { + await withKeyedLock(`obs:${sessionId}`, async () => { const session = await kv.get(KV.sessions, sessionId); if (!session) return; const shaSet = new Set(session.commitShas ?? []); diff --git a/src/triggers/events.ts b/src/triggers/events.ts index 1e2482360b..c3187ce5d4 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -2,6 +2,7 @@ import { TriggerAction, type IIIClient } from "iii-sdk"; import type { CompressedObservation, HookPayload, Memory, Session } from "../types.js"; import { KV, STREAM } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { isReflectEnabled } from "../functions/slots.js"; import { detectLlmProviderKind, @@ -56,7 +57,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { ? data.agentId.trim().slice(0, 128) : undefined; const agentId = requestAgentId ?? getAgentId(); - const session: Session = { + const freshSession: Session = { id: data.sessionId, project: data.project, cwd: data.cwd, @@ -65,7 +66,18 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { observationCount: 0, ...(agentId ? { agentId } : {}), }; - await kv.set(KV.sessions, data.sessionId, session); + const session = await withKeyedLock(`obs:${data.sessionId}`, async () => { + const existing = await kv.get(KV.sessions, data.sessionId); + const merged: Session = { + ...freshSession, + observationCount: existing?.observationCount ?? freshSession.observationCount, + firstPrompt: existing?.firstPrompt, + summary: existing?.summary, + commitShas: existing?.commitShas, + }; + await kv.set(KV.sessions, data.sessionId, merged); + return merged; + }); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } @@ -160,10 +172,12 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction( "event::session::ended", async (data: { sessionId: string }) => { - await kv.update(KV.sessions, data.sessionId, [ - { type: "set", path: "endedAt", value: new Date().toISOString() }, - { type: "set", path: "status", value: "completed" }, - ]); + await withKeyedLock(`obs:${data.sessionId}`, () => + kv.update(KV.sessions, data.sessionId, [ + { type: "set", path: "endedAt", value: new Date().toISOString() }, + { type: "set", path: "status", value: "completed" }, + ]), + ); return { success: true }; }, ); diff --git a/test/graph-cross-path-locks.test.ts b/test/graph-cross-path-locks.test.ts new file mode 100644 index 0000000000..cbab748db2 --- /dev/null +++ b/test/graph-cross-path-locks.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { persistGraphDelta } from "../src/functions/graph.js"; +import { registerCascadeFunction } from "../src/functions/cascade.js"; +import { KV } from "../src/state/schema.js"; +import type { GraphNode, GraphSnapshot, Memory } from "../src/types.js"; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: { + scope: string; + key: string; + release: () => void; + promise: Promise; + } | null = null; + + return { + store, + armGetGate: (scope: string, key: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, release, promise }; + return release; + }, + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + await g.promise; + } + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + return m ? clone(Array.from(m.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const fns = new Map Promise>(); + return { + registerFunction: (id: string, handler: (data: unknown) => Promise) => { + fns.set(id, handler); + }, + trigger: (id: string, data: unknown) => fns.get(id)!(data), + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("graph:persist serializes cross-path graph mutations", () => { + it("does not lose cascade's stale flag or persistGraphDelta's merge when they race on the same node", async () => { + const kv = mockKV(); + + const node: GraphNode = { + id: "node_1", + type: "concept", + name: "react", + properties: {}, + sourceObservationIds: ["obs_old"], + createdAt: "2026-03-01T00:00:00Z", + }; + await kv.set(KV.graphNodes, "node_1", node); + await kv.set(KV.graphNameIndex, "concept|react", "node_1"); + await kv.set(KV.graphNodeDegree, "node_1", 0); + await kv.set(KV.graphSnapshot, "current", { + version: 1, + topNodes: [node], + topEdges: [], + topDegrees: {}, + stats: { + totalNodes: 1, + totalEdges: 0, + nodesByType: { concept: 1 }, + edgesByType: {}, + }, + updatedAt: new Date().toISOString(), + dirty: false, + } satisfies GraphSnapshot); + + const supersededMemory: Memory = { + id: "mem_old", + createdAt: "2026-03-01T00:00:00Z", + updatedAt: "2026-03-01T00:00:00Z", + type: "fact", + title: "Old fact", + content: "Old content", + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: false, + sourceObservationIds: ["obs_old"], + }; + await kv.set(KV.memories, "mem_old", supersededMemory); + + const sdk = mockSdk(); + registerCascadeFunction(sdk as never, kv as never); + + const releaseNodeRowRead = kv.armGetGate(KV.graphNodes, "node_1"); + + const incomingNode: GraphNode = { + id: "node_new", + type: "concept", + name: "react", + properties: {}, + sourceObservationIds: ["obs_new"], + createdAt: new Date().toISOString(), + }; + const persistPromise = persistGraphDelta( + kv as never, + [incomingNode], + [], + ["obs_new"], + ); + + await flush(); + + const cascadePromise = sdk.trigger("mem::cascade-update", { + supersededMemoryId: "mem_old", + }) as Promise<{ success: boolean; flagged: { nodes: number } }>; + + await flush(); + releaseNodeRowRead(); + + const [, cascadeResult] = await Promise.all([persistPromise, cascadePromise]); + + expect(cascadeResult.success).toBe(true); + expect(cascadeResult.flagged.nodes).toBe(1); + + const finalNode = await kv.get(KV.graphNodes, "node_1"); + expect(finalNode!.stale).toBe(true); + expect(finalNode!.sourceObservationIds.sort()).toEqual(["obs_new", "obs_old"]); + }); +}); diff --git a/test/graph-extract-concurrency.test.ts b/test/graph-extract-concurrency.test.ts new file mode 100644 index 0000000000..ca022a66a1 --- /dev/null +++ b/test/graph-extract-concurrency.test.ts @@ -0,0 +1,225 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { persistGraphDelta, registerGraphFunction } from "../src/functions/graph.js"; +import { KV } from "../src/state/schema.js"; +import type { GraphNode, GraphEdge, GraphSnapshot } from "../src/types.js"; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: { + scope: string; + key: string; + release: () => void; + promise: Promise; + } | null = null; + let listGate: { scope: string; release: () => void; promise: Promise } | null = null; + + return { + store, + armListGate: (scope: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + listGate = { scope, release, promise }; + return release; + }, + armGetGate: (scope: string, key: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, release, promise }; + return release; + }, + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + await g.promise; + } + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + const values = m ? clone(Array.from(m.values()) as T[]) : []; + if (listGate && listGate.scope === scope) { + const g = listGate; + listGate = null; + await g.promise; + } + return values; + }, + }; +} + +function node(overrides: Partial & { id: string; name: string }): GraphNode { + return { + type: "file", + properties: {}, + sourceObservationIds: [], + createdAt: new Date().toISOString(), + ...overrides, + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("persistGraphDelta concurrent merges (finding 7)", () => { + it("does not create duplicate nodes when two concurrent extractions resolve the same (type, name) key", async () => { + const kv = mockKV(); + + const nameIndexKey = "file|src/index.ts"; + const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, nameIndexKey); + + const nodeA = node({ + id: "node_a", + name: "src/index.ts", + sourceObservationIds: ["obs_a"], + }); + const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_a"]); + + await flush(); + + const nodeB = node({ + id: "node_b", + name: "src/index.ts", + sourceObservationIds: ["obs_b"], + }); + const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_b"]); + + await flush(); + releaseNameIndexRead(); + + await Promise.all([promiseA, promiseB]); + + const allNodes = await kv.list(KV.graphNodes); + expect(allNodes.length).toBe(1); + + const merged = allNodes[0]; + expect(merged.sourceObservationIds.sort()).toEqual(["obs_a", "obs_b"]); + + const snapshot = (await kv.get( + KV.graphSnapshot, + "current", + )) as GraphSnapshot; + expect(snapshot.stats.totalNodes).toBe(1); + }); + + it("does not lose either batch's obsIds when merging into an existing node concurrently", async () => { + const kv = mockKV(); + + const existing: GraphNode = node({ + id: "node_existing", + name: "src/utils.ts", + sourceObservationIds: ["obs_seed"], + }); + await kv.set(KV.graphNodes, "node_existing", existing); + await kv.set(KV.graphNameIndex, "file|src/utils.ts", "node_existing"); + await kv.set(KV.graphNodeDegree, "node_existing", 0); + await kv.set(KV.graphSnapshot, "current", { + version: 1, + topNodes: [existing], + topEdges: [], + topDegrees: {}, + stats: { + totalNodes: 1, + totalEdges: 0, + nodesByType: { file: 1 }, + edgesByType: {}, + }, + updatedAt: new Date().toISOString(), + dirty: false, + } satisfies GraphSnapshot); + + const releaseExistingNodeRead = kv.armGetGate(KV.graphNodes, "node_existing"); + + const nodeA = node({ + id: "node_a2", + name: "src/utils.ts", + sourceObservationIds: ["obs_x"], + }); + const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_x"]); + + await flush(); + + const nodeB = node({ + id: "node_b2", + name: "src/utils.ts", + sourceObservationIds: ["obs_y"], + }); + const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_y"]); + + await flush(); + releaseExistingNodeRead(); + + await Promise.all([promiseA, promiseB]); + + const allNodes = await kv.list(KV.graphNodes); + expect(allNodes.length).toBe(1); + expect(allNodes[0].sourceObservationIds.sort()).toEqual([ + "obs_seed", + "obs_x", + "obs_y", + ]); + }); + + it("snapshot rebuild does not overwrite a delta that commits while it enumerates", async () => { + const kv = mockKV(); + const handlers = new Map Promise>(); + const sdk = { + registerFunction: (id: string, fn: (data?: unknown) => Promise) => { + handlers.set(id, fn); + }, + }; + registerGraphFunction(sdk as never, kv as never, { name: "noop" } as never); + + const seeded = node({ id: "node_seed", name: "src/seed.ts" }); + await persistGraphDelta(kv as never, [seeded], [], ["obs_seed"]); + + const releaseNodeList = kv.armListGate(KV.graphNodes); + const rebuild = handlers.get("mem::graph-snapshot-rebuild")!(); + await flush(); + + const fresh = node({ id: "node_fresh", name: "src/fresh.ts" }); + const delta = persistGraphDelta(kv as never, [fresh], [], ["obs_fresh"]); + await flush(); + releaseNodeList(); + + const rebuilt = (await rebuild) as { success: boolean }; + await delta; + + expect(rebuilt.success).toBe(true); + const snapshot = (await kv.get( + KV.graphSnapshot, + "current", + )) as GraphSnapshot; + expect(snapshot.stats.totalNodes).toBe(2); + expect(snapshot.topNodes.map((n) => n.id).sort()).toEqual([ + "node_fresh", + "node_seed", + ]); + }); +}); diff --git a/test/session-kv-races.test.ts b/test/session-kv-races.test.ts new file mode 100644 index 0000000000..40c2b0f10c --- /dev/null +++ b/test/session-kv-races.test.ts @@ -0,0 +1,295 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerObserveFunction } from "../src/functions/observe.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; +import { registerEventTriggers } from "../src/triggers/events.js"; +import { KV } from "../src/state/schema.js"; +import type { Session } from "../src/types.js"; + +type Gate = { scope: string; key: string; phase: "get" | "set" }; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: (Gate & { release: () => void; promise: Promise }) | null = null; + + function arm(scope: string, key: string, phase: "get" | "set"): () => void { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, phase, release, promise }; + return release; + } + + async function maybeWait(scope: string, key: string, phase: "get" | "set") { + if (gate && gate.scope === scope && gate.key === key && gate.phase === phase) { + const g = gate; + gate = null; + await g.promise; + } + } + + return { + store, + armGetGate: (scope: string, key: string) => arm(scope, key, "get"), + armSetGate: (scope: string, key: string) => arm(scope, key, "set"), + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + await maybeWait(scope, key, "get"); + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + await maybeWait(scope, key, "set"); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + update: async ( + scope: string, + key: string, + ops: Array<{ type: string; path: string; value?: unknown }>, + ) => { + if (!store.has(scope)) store.set(scope, new Map()); + const m = store.get(scope)!; + const current = clone((m.get(key) as Record) ?? {}); + for (const op of ops) { + if (op.type === "set") current[op.path] = op.value; + } + m.set(key, current); + return clone(current); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + return m ? clone(Array.from(m.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const fns = new Map(); + return { + fns, + registerFunction: (idOrOpts: string | { id: string }, fn: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + fns.set(id, fn); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown; action?: unknown }, + data?: unknown, + ) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = fns.get(id); + if (fn) return fn(payload); + return null; + }, + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("KV.sessions concurrent mutation races (finding 3)", () => { + it("api::session::commit does not revert a concurrent mem::observe's observationCount increment", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + + const session: Session = { + id: "ses_race1", + project: "/repo", + cwd: "/repo", + startedAt: new Date().toISOString(), + status: "active", + observationCount: 0, + }; + await kv.set(KV.sessions, "ses_race1", session); + + const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race1"); + + const commitPromise = sdk.trigger("api::session::commit", { + body: { sha: "abc123def", sessionId: "ses_race1" }, + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race1", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "do the thing" }, + }); + + await flush(); + releaseCommitRead(); + + await Promise.all([commitPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race1") as Session; + expect(finalSession.observationCount).toBe(1); + expect(finalSession.commitShas).toEqual(["abc123def"]); + }); + + it("mem::observe's implicit session create does not clobber a concurrent api::session::start's title", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + const releaseStartWrite = kv.armSetGate(KV.sessions, "ses_race2"); + + const startPromise = sdk.trigger("api::session::start", { + body: { + sessionId: "ses_race2", + project: "/repo", + cwd: "/repo", + title: "fix the parser", + }, + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race2", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "fix the parser" }, + }); + + await flush(); + releaseStartWrite(); + + await Promise.all([startPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race2") as Session; + expect(finalSession.summary).toBe("fix the parser"); + expect(finalSession.observationCount).toBe(1); + }); + + it("event::session::started racing mem::observe's implicit create does not revert the observation count to zero", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEventTriggers(sdk as never, kv as never); + registerObserveFunction(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + const releaseEventWrite = kv.armSetGate(KV.sessions, "ses_race3"); + + const eventPromise = sdk.trigger("event::session::started", { + sessionId: "ses_race3", + project: "/repo", + cwd: "/repo", + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race3", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "start work" }, + }); + + await flush(); + releaseEventWrite(); + + await Promise.all([eventPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race3") as Session; + expect(finalSession).toBeTruthy(); + expect(finalSession.observationCount).toBe(1); + }); + + it("api::session::commit does not revert a concurrent api::session::end status update", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never); + + const session: Session = { + id: "ses_race5", + project: "/repo", + cwd: "/repo", + startedAt: new Date().toISOString(), + status: "active", + observationCount: 3, + }; + await kv.set(KV.sessions, "ses_race5", session); + + const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race5"); + + const commitPromise = sdk.trigger("api::session::commit", { + body: { sha: "deadbeef01", sessionId: "ses_race5" }, + }); + + await flush(); + + const endPromise = sdk.trigger("api::session::end", { + body: { sessionId: "ses_race5" }, + }); + + await flush(); + releaseCommitRead(); + + await Promise.all([commitPromise, endPromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race5") as Session; + expect(finalSession.commitShas).toEqual(["deadbeef01"]); + expect(finalSession.status).toBe("completed"); + expect(finalSession.endedAt).toBeTruthy(); + }); + + it("api::session::start does not reset an observation count or firstPrompt already created by mem::observe", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + await sdk.trigger("mem::observe", { + sessionId: "ses_race4", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "do the thing" }, + }); + + await sdk.trigger("api::session::start", { + body: { + sessionId: "ses_race4", + project: "/repo", + cwd: "/repo", + title: "fix the parser", + }, + }); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race4") as Session; + expect(finalSession.observationCount).toBe(1); + expect(finalSession.firstPrompt).toBe("fix the parser"); + expect(finalSession.status).toBe("active"); + }); +});