From 5e0b45930c359d9f08029a44d1f0010791f9285c Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Fri, 25 Sep 2026 20:03:24 +0530 Subject: [PATCH 1/5] fix(state): close the KV.sessions concurrent read-modify-write race Every whole-object writer to KV.sessions (api::session::start, event::session::started, api::session::commit, plus the bulk import paths in migrate/snapshot/replay/export-import) read the session, mutated a local copy, and wrote it back with no lock, or with a lock under a different namespace than mem::observe's obs:${sessionId}. A concurrent kv.update from observe (or from any other writer) landing between the read and the write got silently reverted by the later whole-object set. Route every session mutation through the same obs:${sessionId} keyed lock observe.ts already uses, so no two writers can interleave a read against a write for the same session. api::session::commit keeps its get-then-set shape (it needs the current commitShas to dedupe) but now shares the lock namespace instead of its own session:${sessionId} key, which also keeps it compatible with the pending session::end lock in PR #1252. session::end and event::session::ended already mutate via kv.update path ops, which the engine applies atomically per call, so they are left as-is. Adds test/session-kv-races.test.ts, which reproduces the lost-update race deterministically (a gated mock KV that pauses a read mid-flight so a concurrent writer can land in between) and fails on main before this change. --- src/functions/export-import.ts | 23 ++-- src/functions/migrate.ts | 5 +- src/functions/replay.ts | 81 ++++++------ src/functions/snapshot.ts | 5 +- src/triggers/api.ts | 6 +- src/triggers/events.ts | 5 +- test/session-kv-races.test.ts | 226 +++++++++++++++++++++++++++++++++ 7 files changed, 297 insertions(+), 54 deletions(-) create mode 100644 test/session-kv-races.test.ts diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 6d0128f36a..2118c47ae4 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -29,6 +29,7 @@ import { normalizeAccessLog } from "./access-tracker.js"; import { KV } from "../state/schema.js"; import { checkPayloadFrameSize } from "../state/frame-guard.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { VERSION } from "../version.js"; import { recordAudit } from "./audit.js"; import { indexRecords } from "./search.js"; @@ -407,17 +408,19 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void const indexMems: Memory[] = []; await runChunked(importData.sessions, async (session) => { - if (strategy === "skip") { - const existing = await kv - .get(KV.sessions, session.id) - .catch(() => null); - if (existing) { - stats.skipped++; - return; + await withKeyedLock(`obs:${session.id}`, async () => { + if (strategy === "skip") { + const existing = await kv + .get(KV.sessions, session.id) + .catch(() => null); + if (existing) { + stats.skipped++; + return; + } } - } - await kv.set(KV.sessions, session.id, session); - stats.sessions++; + await kv.set(KV.sessions, session.id, session); + stats.sessions++; + }); }); for (const [sessionId, obs] of Object.entries(importData.observations)) { diff --git a/src/functions/migrate.ts b/src/functions/migrate.ts index 8cabef5315..350a533250 100644 --- a/src/functions/migrate.ts +++ b/src/functions/migrate.ts @@ -3,6 +3,7 @@ import { resolve } from "node:path"; import { homedir } from "node:os"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import type { Memory, Session, @@ -150,7 +151,9 @@ export function registerMigrateFunction(sdk: IIIClient, kv: StateKV): void { status: "completed", observationCount: 0, }; - await kv.set(KV.sessions, session.id, session); + await withKeyedLock(`obs:${session.id}`, () => + kv.set(KV.sessions, session.id, session), + ); sessionCount++; } diff --git a/src/functions/replay.ts b/src/functions/replay.ts index aea735ff98..d158b91384 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -12,6 +12,7 @@ import type { import { importOrigin } from "../types.js"; import type { StateKV } from "../state/kv.js"; import { KV, generateId, fingerprintId } from "../state/schema.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { parseJsonlText } from "../replay/jsonl-parser.js"; import { resetLessonIndex } from "./lessons.js"; import { projectTimeline, type Timeline } from "../replay/timeline.js"; @@ -394,46 +395,48 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void { ? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) : undefined; - const existing = await kv.get(KV.sessions, parsed.sessionId); - if (existing) { - existing.observationCount = - (existing.observationCount || 0) + parsed.observations.length; - if (parsed.endedAt > (existing.endedAt || "")) { - existing.endedAt = parsed.endedAt; + await withKeyedLock(`obs:${parsed.sessionId}`, async () => { + const existing = await kv.get(KV.sessions, parsed.sessionId); + if (existing) { + existing.observationCount = + (existing.observationCount || 0) + parsed.observations.length; + if (parsed.endedAt > (existing.endedAt || "")) { + existing.endedAt = parsed.endedAt; + } + if (existing.status === "active") existing.status = "completed"; + const existingTags = existing.tags || []; + if (!existingTags.includes("jsonl-import")) { + existing.tags = [...existingTags, "jsonl-import"]; + } + if (!existing.firstPrompt && firstPrompt) { + existing.firstPrompt = firstPrompt; + } + // #775: re-key on parsed.sessionId, not existing.id. Older + // session rows may be missing the `id` field; existing.id + // would then be undefined, JSON.stringify would drop the + // `key` from the state::set payload, and the engine would + // reject the call with `missing field \`key\``. Because the + // rejection aborts the whole import handler, a single + // legacy row killed the entire batch. parsed.sessionId is + // always populated (parseJsonlText has a three-level + // fallback) and is what we just used to read the row. + if (!existing.id) existing.id = parsed.sessionId; + await kv.set(KV.sessions, parsed.sessionId, existing); + } else { + const session: Session = { + id: parsed.sessionId, + project: parsed.project, + cwd: parsed.cwd, + startedAt: parsed.startedAt, + endedAt: parsed.endedAt, + status: "completed", + observationCount: parsed.observations.length, + tags: ["jsonl-import"], + firstPrompt, + }; + await kv.set(KV.sessions, session.id, session); } - if (existing.status === "active") existing.status = "completed"; - const existingTags = existing.tags || []; - if (!existingTags.includes("jsonl-import")) { - existing.tags = [...existingTags, "jsonl-import"]; - } - if (!existing.firstPrompt && firstPrompt) { - existing.firstPrompt = firstPrompt; - } - // #775: re-key on parsed.sessionId, not existing.id. Older - // session rows may be missing the `id` field; existing.id - // would then be undefined, JSON.stringify would drop the - // `key` from the state::set payload, and the engine would - // reject the call with `missing field \`key\``. Because the - // rejection aborts the whole import handler, a single - // legacy row killed the entire batch. parsed.sessionId is - // always populated (parseJsonlText has a three-level - // fallback) and is what we just used to read the row. - if (!existing.id) existing.id = parsed.sessionId; - await kv.set(KV.sessions, parsed.sessionId, existing); - } else { - const session: Session = { - id: parsed.sessionId, - project: parsed.project, - cwd: parsed.cwd, - startedAt: parsed.startedAt, - endedAt: parsed.endedAt, - status: "completed", - observationCount: parsed.observations.length, - tags: ["jsonl-import"], - firstPrompt, - }; - await kv.set(KV.sessions, session.id, session); - } + }); const compressed: CompressedObservation[] = []; await Promise.all( diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 483240e14d..9d5594046e 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -12,6 +12,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; import { VERSION } from "../version.js"; import { logger } from "../logger.js"; @@ -195,7 +196,9 @@ export function registerSnapshotFunction( if (state.sessions) { for (const session of state.sessions) { - await kv.set(KV.sessions, session.id, session); + await withKeyedLock(`obs:${session.id}`, () => + kv.set(KV.sessions, session.id, session), + ); } } if (state.memories) { diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d8a447f2e0..62ea99a93f 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -732,7 +732,9 @@ export function registerApiTriggers( ...(title ? { firstPrompt: title.slice(0, 200) } : {}), ...(agentId ? { agentId } : {}), }; - await kv.set(KV.sessions, sessionId, session); + await withKeyedLock(`obs:${sessionId}`, () => + kv.set(KV.sessions, sessionId, session), + ); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } @@ -861,7 +863,7 @@ export function registerApiTriggers( }); if (sessionId) { - await withKeyedLock(`session:${sessionId}`, async () => { + await withKeyedLock(`obs:${sessionId}`, async () => { const session = await kv.get(KV.sessions, sessionId); if (!session) return; const shaSet = new Set(session.commitShas ?? []); diff --git a/src/triggers/events.ts b/src/triggers/events.ts index 1e2482360b..a8d8895da7 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -2,6 +2,7 @@ import { TriggerAction, type IIIClient } from "iii-sdk"; import type { CompressedObservation, HookPayload, Memory, Session } from "../types.js"; import { KV, STREAM } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { isReflectEnabled } from "../functions/slots.js"; import { detectLlmProviderKind, @@ -65,7 +66,9 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { observationCount: 0, ...(agentId ? { agentId } : {}), }; - await kv.set(KV.sessions, data.sessionId, session); + await withKeyedLock(`obs:${data.sessionId}`, () => + kv.set(KV.sessions, data.sessionId, session), + ); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } diff --git a/test/session-kv-races.test.ts b/test/session-kv-races.test.ts new file mode 100644 index 0000000000..dbb7614acb --- /dev/null +++ b/test/session-kv-races.test.ts @@ -0,0 +1,226 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerObserveFunction } from "../src/functions/observe.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; +import { registerEventTriggers } from "../src/triggers/events.js"; +import { KV } from "../src/state/schema.js"; +import type { Session } from "../src/types.js"; + +type Gate = { scope: string; key: string; phase: "get" | "set" }; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: (Gate & { release: () => void; promise: Promise }) | null = null; + + function arm(scope: string, key: string, phase: "get" | "set"): () => void { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, phase, release, promise }; + return release; + } + + async function maybeWait(scope: string, key: string, phase: "get" | "set") { + if (gate && gate.scope === scope && gate.key === key && gate.phase === phase) { + const g = gate; + gate = null; + await g.promise; + } + } + + return { + store, + armGetGate: (scope: string, key: string) => arm(scope, key, "get"), + armSetGate: (scope: string, key: string) => arm(scope, key, "set"), + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + await maybeWait(scope, key, "get"); + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + await maybeWait(scope, key, "set"); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + update: async ( + scope: string, + key: string, + ops: Array<{ type: string; path: string; value?: unknown }>, + ) => { + if (!store.has(scope)) store.set(scope, new Map()); + const m = store.get(scope)!; + const current = clone((m.get(key) as Record) ?? {}); + for (const op of ops) { + if (op.type === "set") current[op.path] = op.value; + } + m.set(key, current); + return clone(current); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + return m ? clone(Array.from(m.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const fns = new Map(); + return { + fns, + registerFunction: (idOrOpts: string | { id: string }, fn: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + fns.set(id, fn); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown; action?: unknown }, + data?: unknown, + ) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = fns.get(id); + if (fn) return fn(payload); + return null; + }, + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("KV.sessions concurrent mutation races (finding 3)", () => { + it("api::session::commit does not revert a concurrent mem::observe's observationCount increment", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + + const session: Session = { + id: "ses_race1", + project: "/repo", + cwd: "/repo", + startedAt: new Date().toISOString(), + status: "active", + observationCount: 0, + }; + await kv.set(KV.sessions, "ses_race1", session); + + const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race1"); + + const commitPromise = sdk.trigger("api::session::commit", { + body: { sha: "abc123def", sessionId: "ses_race1" }, + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race1", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "do the thing" }, + }); + + await flush(); + releaseCommitRead(); + + await Promise.all([commitPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race1") as Session; + expect(finalSession.observationCount).toBe(1); + expect(finalSession.commitShas).toEqual(["abc123def"]); + }); + + it("mem::observe's implicit session create does not clobber a concurrent api::session::start's title", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + const releaseStartWrite = kv.armSetGate(KV.sessions, "ses_race2"); + + const startPromise = sdk.trigger("api::session::start", { + body: { + sessionId: "ses_race2", + project: "/repo", + cwd: "/repo", + title: "fix the parser", + }, + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race2", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "fix the parser" }, + }); + + await flush(); + releaseStartWrite(); + + await Promise.all([startPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race2") as Session; + expect(finalSession.summary).toBe("fix the parser"); + expect(finalSession.observationCount).toBe(1); + }); + + it("event::session::started racing mem::observe's implicit create does not revert the observation count to zero", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerEventTriggers(sdk as never, kv as never); + registerObserveFunction(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + const releaseEventWrite = kv.armSetGate(KV.sessions, "ses_race3"); + + const eventPromise = sdk.trigger("event::session::started", { + sessionId: "ses_race3", + project: "/repo", + cwd: "/repo", + }); + + await flush(); + + const observePromise = sdk.trigger("mem::observe", { + sessionId: "ses_race3", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "start work" }, + }); + + await flush(); + releaseEventWrite(); + + await Promise.all([eventPromise, observePromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race3") as Session; + expect(finalSession).toBeTruthy(); + expect(finalSession.observationCount).toBe(1); + }); +}); From aa2502298264013c9725f8843b5a5fbce86c5f8e Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Fri, 25 Sep 2026 20:03:40 +0530 Subject: [PATCH 2/5] fix(graph): serialize persistGraphDelta so concurrent extracts cannot duplicate nodes persistGraphDelta read the graph snapshot once, resolved each node/edge through the name/edge index with a get-then-set (no lock anywhere in graph.ts), and wrote the accumulated snapshot back once at the end. Two concurrent mem::graph-extract calls (routine when session-stops land within seconds of each other) could both miss the same name-index key and each create their own node for the same (type, name) pair, or both merge into the same existing node from a stale copy and drop one side's sourceObservationIds, or overwrite each other's snapshot stat deltas. The node/edge keys aren't known until the name/edge index lookup resolves, and the snapshot is a single shared aggregate mutated in-memory across the whole batch, so a per-key lock alone would still let two batches race the aggregate snapshot write. Serialize the whole function body behind one graph:persist keyed lock instead, matching mesh.ts's existing per-item locking style for the parts that are per-key. graph-import.ts's structural importer calls the same function, so it gets the same protection for free. Adds test/graph-extract-concurrency.test.ts: two concurrent persistGraphDelta calls for the same (type, name) key must not create duplicate nodes, and two concurrent merges into an existing node must not lose either batch's sourceObservationIds. Both fail on main before this change. --- src/functions/graph.ts | 225 +++++++++++++------------ test/graph-extract-concurrency.test.ts | 173 +++++++++++++++++++ 2 files changed, 287 insertions(+), 111 deletions(-) create mode 100644 test/graph-extract-concurrency.test.ts diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 62e7343e3b..ee4974106c 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -9,6 +9,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { GRAPH_EXTRACTION_SYSTEM, buildGraphExtractionPrompt, @@ -554,129 +555,131 @@ export async function persistGraphDelta( edges: GraphEdge[], obsIds: string[], ): Promise<{ newNodeCount: number; newEdgeCount: number }> { - const snap = (await readSnapshot(kv)) ?? emptySnapshot(); - const capturedAt = new Date().toISOString(); - let newNodeCount = 0; - let newEdgeCount = 0; - // Merge-only batches mutate cached topNodes/topEdges entries without - // changing the counts; track that separately so the snapshot still persists. - let snapMutated = false; - const newEdgesForTopCheck: GraphEdge[] = []; - // When a freshly-minted node merges into an existing row via the name - // index, edges in the same batch still reference the fresh id. Remap edge - // endpoints to the persisted ids so edges never dangle and re-runs hit the - // same edge-index key instead of duplicating. - const idRemap = new Map(); - - for (const node of nodes) { - const indexKey = nameIndexKey(node.type, node.name); - const existingId = await kv.get(KV.graphNameIndex, indexKey); - - let existing: GraphNode | null = null; - if (existingId) { - existing = await kv.get(KV.graphNodes, existingId); - // #825 follow-up: name-index lookups can resolve into - // pre-reset rows. Drop them so extract writes a fresh - // node + index entry instead of silently reconnecting - // to a legacy orphan (which would keep the snapshot at - // 0 forever after a reset). - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; + return withKeyedLock("graph:persist", async () => { + const snap = (await readSnapshot(kv)) ?? emptySnapshot(); + const capturedAt = new Date().toISOString(); + let newNodeCount = 0; + let newEdgeCount = 0; + // Merge-only batches mutate cached topNodes/topEdges entries without + // changing the counts; track that separately so the snapshot still persists. + let snapMutated = false; + const newEdgesForTopCheck: GraphEdge[] = []; + // When a freshly-minted node merges into an existing row via the name + // index, edges in the same batch still reference the fresh id. Remap edge + // endpoints to the persisted ids so edges never dangle and re-runs hit the + // same edge-index key instead of duplicating. + const idRemap = new Map(); + + for (const node of nodes) { + const indexKey = nameIndexKey(node.type, node.name); + const existingId = await kv.get(KV.graphNameIndex, indexKey); + + let existing: GraphNode | null = null; + if (existingId) { + existing = await kv.get(KV.graphNodes, existingId); + // #825 follow-up: name-index lookups can resolve into + // pre-reset rows. Drop them so extract writes a fresh + // node + index entry instead of silently reconnecting + // to a legacy orphan (which would keep the snapshot at + // 0 forever after a reset). + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } } - } - if (existing) { - idRemap.set(node.id, existing.id); - const merged = mergeNode(existing, node, obsIds, capturedAt); - await kv.set(KV.graphNodes, existing.id, merged); - // Update topNodes entry if present so a stale clone isn't - // returned from the snapshot fast path. - const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id); - if (topIdx !== -1) { - snap.topNodes[topIdx] = merged; - snapMutated = true; - } - } else { - await kv.set(KV.graphNodes, node.id, node); - await kv.set(KV.graphNameIndex, indexKey, node.id); - await kv.set(KV.graphNodeDegree, node.id, 0); - snap.stats.totalNodes += 1; - snap.stats.nodesByType[node.type] = - (snap.stats.nodesByType[node.type] ?? 0) + 1; - newNodeCount += 1; - if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { - // Degree 0 still beats an empty slot — sit at the tail - // until edges arrive and promote. - snap.topNodes.push(node); - snap.topDegrees[node.id] = 0; + if (existing) { + idRemap.set(node.id, existing.id); + const merged = mergeNode(existing, node, obsIds, capturedAt); + await kv.set(KV.graphNodes, existing.id, merged); + // Update topNodes entry if present so a stale clone isn't + // returned from the snapshot fast path. + const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id); + if (topIdx !== -1) { + snap.topNodes[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphNodes, node.id, node); + await kv.set(KV.graphNameIndex, indexKey, node.id); + await kv.set(KV.graphNodeDegree, node.id, 0); + snap.stats.totalNodes += 1; + snap.stats.nodesByType[node.type] = + (snap.stats.nodesByType[node.type] ?? 0) + 1; + newNodeCount += 1; + if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { + // Degree 0 still beats an empty slot — sit at the tail + // until edges arrive and promote. + snap.topNodes.push(node); + snap.topDegrees[node.id] = 0; + } } } - } - for (const rawEdge of edges) { - const edge: GraphEdge = { - ...rawEdge, - sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId, - targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId, - }; - const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type); - const existingId = await kv.get(KV.graphEdgeKey, eKey); - - let existing: GraphEdge | null = null; - if (existingId) { - existing = await kv.get(KV.graphEdges, existingId); - // Same #825 orphan check as the node path above. - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; + for (const rawEdge of edges) { + const edge: GraphEdge = { + ...rawEdge, + sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId, + targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId, + }; + const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type); + const existingId = await kv.get(KV.graphEdgeKey, eKey); + + let existing: GraphEdge | null = null; + if (existingId) { + existing = await kv.get(KV.graphEdges, existingId); + // Same #825 orphan check as the node path above. + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } } - } - if (existing) { - const merged = mergeEdge(existing, obsIds); - await kv.set(KV.graphEdges, existing.id, merged); - // Replace cached topEdges entry too if present. - const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); - if (topIdx !== -1) { - snap.topEdges[topIdx] = merged; - snapMutated = true; + if (existing) { + const merged = mergeEdge(existing, obsIds); + await kv.set(KV.graphEdges, existing.id, merged); + // Replace cached topEdges entry too if present. + const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); + if (topIdx !== -1) { + snap.topEdges[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphEdges, edge.id, edge); + await kv.set(KV.graphEdgeKey, eKey, edge.id); + snap.stats.totalEdges += 1; + snap.stats.edgesByType[edge.type] = + (snap.stats.edgesByType[edge.type] ?? 0) + 1; + newEdgeCount += 1; + await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); + await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); + newEdgesForTopCheck.push(edge); } - } else { - await kv.set(KV.graphEdges, edge.id, edge); - await kv.set(KV.graphEdgeKey, eKey, edge.id); - snap.stats.totalEdges += 1; - snap.stats.edgesByType[edge.type] = - (snap.stats.edgesByType[edge.type] ?? 0) + 1; - newEdgeCount += 1; - await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); - await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); - newEdgesForTopCheck.push(edge); } - } - // Push newly-added edges into snapshot.topEdges if both - // endpoints are in the top-N (post-degree-delta). Done after - // all degree updates so the topIds set is stable. - for (const edge of newEdgesForTopCheck) { - snapshotPushEdgeIfBothInTop(snap, edge); - } + // Push newly-added edges into snapshot.topEdges if both + // endpoints are in the top-N (post-degree-delta). Done after + // all degree updates so the topIds set is stable. + for (const edge of newEdgesForTopCheck) { + snapshotPushEdgeIfBothInTop(snap, edge); + } - if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) { - snap.updatedAt = capturedAt; - snap.dirty = false; - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); - } + if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) { + snap.updatedAt = capturedAt; + snap.dirty = false; + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); + } - return { newNodeCount, newEdgeCount }; + return { newNodeCount, newEdgeCount }; + }); } export function registerGraphFunction( diff --git a/test/graph-extract-concurrency.test.ts b/test/graph-extract-concurrency.test.ts new file mode 100644 index 0000000000..ef3ffb4893 --- /dev/null +++ b/test/graph-extract-concurrency.test.ts @@ -0,0 +1,173 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { persistGraphDelta } from "../src/functions/graph.js"; +import { KV } from "../src/state/schema.js"; +import type { GraphNode, GraphEdge, GraphSnapshot } from "../src/types.js"; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: { + scope: string; + key: string; + release: () => void; + promise: Promise; + } | null = null; + + return { + store, + armGetGate: (scope: string, key: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, release, promise }; + return release; + }, + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + await g.promise; + } + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + return m ? clone(Array.from(m.values()) as T[]) : []; + }, + }; +} + +function node(overrides: Partial & { id: string; name: string }): GraphNode { + return { + type: "file", + properties: {}, + sourceObservationIds: [], + createdAt: new Date().toISOString(), + ...overrides, + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("persistGraphDelta concurrent merges (finding 7)", () => { + it("does not create duplicate nodes when two concurrent extractions resolve the same (type, name) key", async () => { + const kv = mockKV(); + + const nameIndexKey = "file|src/index.ts"; + const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, nameIndexKey); + + const nodeA = node({ + id: "node_a", + name: "src/index.ts", + sourceObservationIds: ["obs_a"], + }); + const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_a"]); + + await flush(); + + const nodeB = node({ + id: "node_b", + name: "src/index.ts", + sourceObservationIds: ["obs_b"], + }); + const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_b"]); + + await flush(); + releaseNameIndexRead(); + + await Promise.all([promiseA, promiseB]); + + const allNodes = await kv.list(KV.graphNodes); + expect(allNodes.length).toBe(1); + + const merged = allNodes[0]; + expect(merged.sourceObservationIds.sort()).toEqual(["obs_a", "obs_b"]); + + const snapshot = (await kv.get( + KV.graphSnapshot, + "current", + )) as GraphSnapshot; + expect(snapshot.stats.totalNodes).toBe(1); + }); + + it("does not lose either batch's obsIds when merging into an existing node concurrently", async () => { + const kv = mockKV(); + + const existing: GraphNode = node({ + id: "node_existing", + name: "src/utils.ts", + sourceObservationIds: ["obs_seed"], + }); + await kv.set(KV.graphNodes, "node_existing", existing); + await kv.set(KV.graphNameIndex, "file|src/utils.ts", "node_existing"); + await kv.set(KV.graphNodeDegree, "node_existing", 0); + await kv.set(KV.graphSnapshot, "current", { + version: 1, + topNodes: [existing], + topEdges: [], + topDegrees: {}, + stats: { + totalNodes: 1, + totalEdges: 0, + nodesByType: { file: 1 }, + edgesByType: {}, + }, + updatedAt: new Date().toISOString(), + dirty: false, + } satisfies GraphSnapshot); + + const releaseExistingNodeRead = kv.armGetGate(KV.graphNodes, "node_existing"); + + const nodeA = node({ + id: "node_a2", + name: "src/utils.ts", + sourceObservationIds: ["obs_x"], + }); + const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_x"]); + + await flush(); + + const nodeB = node({ + id: "node_b2", + name: "src/utils.ts", + sourceObservationIds: ["obs_y"], + }); + const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_y"]); + + await flush(); + releaseExistingNodeRead(); + + await Promise.all([promiseA, promiseB]); + + const allNodes = await kv.list(KV.graphNodes); + expect(allNodes.length).toBe(1); + expect(allNodes[0].sourceObservationIds.sort()).toEqual([ + "obs_seed", + "obs_x", + "obs_y", + ]); + }); +}); From cab7a4fc21b932eab53badfff1301f718fc5a547 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 11:47:12 +0530 Subject: [PATCH 3/5] fix(graph): serialize graph-row writers outside persistGraphDelta on graph:persist persistGraphDelta already serializes mem::graph-extract under the graph:persist keyed lock, but temporal extraction, mesh sync, export/import, snapshot restore, and cascade-update all read and wrote KV.graphNodes/KV.graphEdges without it. A concurrent call from any of these paths could overlap persistGraphDelta's read-modify-write and drop the loser's changes. Route each of those write paths through the same graph:persist lock (sibling acquisitions, never nested, since withKeyedLock is not re-entrant): temporal-graph-extract's merge loop, mesh's lwwMergeGraphNodes and a new lwwMergeGraphEdges (replacing the generic per-item lock for graph edges), export/import's graph delete and import loops, snapshot-restore's graph node writes, and cascade-update's stale-flag writes (which now also re-read the latest row inside the lock before writing, instead of writing back the pre-lock in-memory copy). Snapshot/index maintenance (name index, edge index, degree, the graph:snapshot cache) is still only performed by persistGraphDelta and the explicit rebuild/reset functions; that gap predates this change and is out of scope here. This commit only closes the lost-update race on the raw node/edge rows. Added test/graph-cross-path-locks.test.ts, which races persistGraphDelta against mem::cascade-update on the same node and asserts both the merge and the stale flag survive. --- src/functions/cascade.ts | 24 +++-- src/functions/export-import.ts | 38 ++++--- src/functions/mesh.ts | 33 +++++- src/functions/snapshot.ts | 4 +- src/functions/temporal-graph.ts | 137 +++++++++++------------ test/graph-cross-path-locks.test.ts | 161 ++++++++++++++++++++++++++++ 6 files changed, 303 insertions(+), 94 deletions(-) create mode 100644 test/graph-cross-path-locks.test.ts diff --git a/src/functions/cascade.ts b/src/functions/cascade.ts index 2167c506d5..b29596764b 100644 --- a/src/functions/cascade.ts +++ b/src/functions/cascade.ts @@ -1,6 +1,7 @@ import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import type { Memory, GraphNode, GraphEdge } from "../types.js"; import { recordAudit } from "./audit.js"; @@ -28,10 +29,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void { for (const node of nodes) { if (node.stale) continue; const overlap = (node.sourceObservationIds ?? []).some((id) => obsIds.has(id)); - if (overlap) { - node.stale = true; - node.updatedAt = now; - await kv.set(KV.graphNodes, node.id, node); + if (!overlap) continue; + const flagged = await withKeyedLock("graph:persist", async () => { + const latest = (await kv.get(KV.graphNodes, node.id)) ?? node; + if (latest.stale) return false; + await kv.set(KV.graphNodes, node.id, { ...latest, stale: true, updatedAt: now }); + return true; + }); + if (flagged) { await recordAudit(kv, "consolidate", "mem::cascade-update", [node.id], { resourceType: "GraphNode", change: "marked stale from superseded memory", @@ -45,9 +50,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void { for (const edge of edges) { if (edge.stale) continue; const overlap = (edge.sourceObservationIds ?? []).some((id) => obsIds.has(id)); - if (overlap) { - edge.stale = true; - await kv.set(KV.graphEdges, edge.id, edge); + if (!overlap) continue; + const flagged = await withKeyedLock("graph:persist", async () => { + const latest = (await kv.get(KV.graphEdges, edge.id)) ?? edge; + if (latest.stale) return false; + await kv.set(KV.graphEdges, edge.id, { ...latest, stale: true }); + return true; + }); + if (flagged) { await recordAudit(kv, "consolidate", "mem::cascade-update", [edge.id], { resourceType: "GraphEdge", change: "marked stale from superseded memory", diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 2118c47ae4..9badf6003a 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -375,11 +375,13 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void ); await runChunked( await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), - (n) => kv.delete(KV.graphNodes, n.id), + (n) => + withKeyedLock("graph:persist", () => kv.delete(KV.graphNodes, n.id)), ); await runChunked( await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), - (e) => kv.delete(KV.graphEdges, e.id), + (e) => + withKeyedLock("graph:persist", () => kv.delete(KV.graphEdges, e.id)), ); await runChunked( await kv.list<{ id: string }>(KV.semantic).catch(() => []), @@ -476,22 +478,26 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void }); if (importData.graphNodes) { - await runChunked(importData.graphNodes, async (node) => { - if (strategy === "skip") { - const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); - if (existing) { stats.skipped++; return; } - } - await kv.set(KV.graphNodes, node.id, node); - }); + await runChunked(importData.graphNodes, (node) => + withKeyedLock("graph:persist", async () => { + if (strategy === "skip") { + const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); + if (existing) { stats.skipped++; return; } + } + await kv.set(KV.graphNodes, node.id, node); + }), + ); } if (importData.graphEdges) { - await runChunked(importData.graphEdges, async (edge) => { - if (strategy === "skip") { - const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); - if (existing) { stats.skipped++; return; } - } - await kv.set(KV.graphEdges, edge.id, edge); - }); + await runChunked(importData.graphEdges, (edge) => + withKeyedLock("graph:persist", async () => { + if (strategy === "skip") { + const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); + if (existing) { stats.skipped++; return; } + } + await kv.set(KV.graphEdges, edge.id, edge); + }), + ); } if (importData.semanticMemories) { await runChunked(importData.semanticMemories, async (sem) => { diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts index 139765adf2..0dd7e95753 100644 --- a/src/functions/mesh.ts +++ b/src/functions/mesh.ts @@ -119,7 +119,7 @@ async function lwwMergeGraphNodes( if (!item.id || typeof item.id !== "string") continue; const ts = graphNodeTs(item); if (!ts || Number.isNaN(new Date(ts).getTime())) continue; - const wrote = await withKeyedLock(`mem:gnode:${item.id}`, async () => { + const wrote = await withKeyedLock("graph:persist", async () => { const existing = await kv.get(KV.graphNodes, item.id); if (!existing) { await kv.set(KV.graphNodes, item.id, item); @@ -136,6 +136,33 @@ async function lwwMergeGraphNodes( return count; } +async function lwwMergeGraphEdges( + kv: StateKV, + items: GraphEdge[] | undefined, +): Promise { + if (!items || !Array.isArray(items)) return 0; + let count = 0; + for (const item of items) { + if (!item.id || typeof item.id !== "string") continue; + const ts = item.createdAt; + if (!ts || Number.isNaN(new Date(ts).getTime())) continue; + const wrote = await withKeyedLock("graph:persist", async () => { + const existing = await kv.get(KV.graphEdges, item.id); + if (!existing) { + await kv.set(KV.graphEdges, item.id, item); + return true; + } + if (new Date(ts) > new Date(existing.createdAt)) { + await kv.set(KV.graphEdges, item.id, item); + return true; + } + return false; + }); + if (wrote) count++; + } + return count; +} + export function registerMeshFunction( sdk: IIIClient, kv: StateKV, @@ -361,7 +388,7 @@ export function registerMeshFunction( } } accepted += await lwwMergeGraphNodes(kv, data.graphNodes); - accepted += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt"); + accepted += await lwwMergeGraphEdges(kv, data.graphEdges); await recordAudit(kv, "mesh_sync", "mem::mesh-receive", [], { action: "mesh.receive", accepted, @@ -488,7 +515,7 @@ async function applySyncData( applied += await lwwMergeGraphNodes(kv, data.graphNodes); } if (scopes.includes("graph:edges")) { - applied += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt"); + applied += await lwwMergeGraphEdges(kv, data.graphEdges); } return applied; diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 9d5594046e..066738451c 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -208,7 +208,9 @@ export function registerSnapshotFunction( } if (state.graphNodes) { for (const node of state.graphNodes) { - await kv.set(KV.graphNodes, node.id, node); + await withKeyedLock("graph:persist", () => + kv.set(KV.graphNodes, node.id, node), + ); } } if (state.observations) { diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index aed6730b5f..18d13c1fe5 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -9,6 +9,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; const TEMPORAL_EXTRACTION_SYSTEM = `You are a temporal knowledge extraction engine. Given observations, extract entities AND their temporal relationships with full context metadata. @@ -186,76 +187,78 @@ export function registerTemporalGraphFunctions( const obsIds = data.observations.map((o) => o.id); const { nodes, edges } = parseTemporalGraphXml(response, obsIds); - const existingNodes = await kv.list(KV.graphNodes); - const existingEdges = await kv.list(KV.graphEdges); - - const idRemap = new Map(); - for (const node of nodes) { - const existing = existingNodes.find( - (n) => - n.name === node.name && n.type === node.type, - ); - if (existing) { - const oldId = node.id; - const merged = { - ...existing, - sourceObservationIds: [ - ...new Set([ - ...existing.sourceObservationIds, - ...obsIds, - ]), - ], - properties: { ...existing.properties, ...node.properties }, - updatedAt: new Date().toISOString(), - aliases: [ - ...new Set([ - ...(existing.aliases || []), - ...(node.aliases || []), - ]), - ], - }; - if (merged.aliases.length === 0) delete (merged as any).aliases; - await kv.set(KV.graphNodes, existing.id, merged); - node.id = existing.id; - idRemap.set(oldId, existing.id); - } else { - await kv.set(KV.graphNodes, node.id, node); - existingNodes.push(node); + await withKeyedLock("graph:persist", async () => { + const existingNodes = await kv.list(KV.graphNodes); + const existingEdges = await kv.list(KV.graphEdges); + + const idRemap = new Map(); + for (const node of nodes) { + const existing = existingNodes.find( + (n) => + n.name === node.name && n.type === node.type, + ); + if (existing) { + const oldId = node.id; + const merged = { + ...existing, + sourceObservationIds: [ + ...new Set([ + ...existing.sourceObservationIds, + ...obsIds, + ]), + ], + properties: { ...existing.properties, ...node.properties }, + updatedAt: new Date().toISOString(), + aliases: [ + ...new Set([ + ...(existing.aliases || []), + ...(node.aliases || []), + ]), + ], + }; + if (merged.aliases.length === 0) delete (merged as any).aliases; + await kv.set(KV.graphNodes, existing.id, merged); + node.id = existing.id; + idRemap.set(oldId, existing.id); + } else { + await kv.set(KV.graphNodes, node.id, node); + existingNodes.push(node); + } } - } - for (const edge of edges) { - if (idRemap.has(edge.sourceNodeId)) { - edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!; + for (const edge of edges) { + if (idRemap.has(edge.sourceNodeId)) { + edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!; + } + if (idRemap.has(edge.targetNodeId)) { + edge.targetNodeId = idRemap.get(edge.targetNodeId)!; + } + const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`; + const existingEdge = existingEdges.find( + (e) => + `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` === + existingKey, + ); + + if (existingEdge) { + const updatedOld = { + ...existingEdge, + isLatest: false, + tvalidEnd: + existingEdge.tvalidEnd || new Date().toISOString(), + supersededBy: edge.id, + }; + await kv.set(KV.graphEdges, existingEdge.id, updatedOld); + + await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld); + + edge.version = (existingEdge.version || 1) + 1; + } + + await kv.set(KV.graphEdges, edge.id, edge); + existingEdges.push(edge); } - if (idRemap.has(edge.targetNodeId)) { - edge.targetNodeId = idRemap.get(edge.targetNodeId)!; - } - const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`; - const existingEdge = existingEdges.find( - (e) => - `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` === - existingKey, - ); - - if (existingEdge) { - const updatedOld = { - ...existingEdge, - isLatest: false, - tvalidEnd: - existingEdge.tvalidEnd || new Date().toISOString(), - supersededBy: edge.id, - }; - await kv.set(KV.graphEdges, existingEdge.id, updatedOld); - - await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld); - - edge.version = (existingEdge.version || 1) + 1; - } - - await kv.set(KV.graphEdges, edge.id, edge); - existingEdges.push(edge); - } + }); logger.info("Temporal graph extraction complete", { nodes: nodes.length, diff --git a/test/graph-cross-path-locks.test.ts b/test/graph-cross-path-locks.test.ts new file mode 100644 index 0000000000..c289506030 --- /dev/null +++ b/test/graph-cross-path-locks.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { persistGraphDelta } from "../src/functions/graph.js"; +import { registerCascadeFunction } from "../src/functions/cascade.js"; +import { KV } from "../src/state/schema.js"; +import type { GraphNode, GraphSnapshot, Memory } from "../src/types.js"; + +function clone(value: T): T { + return value === null || value === undefined + ? value + : (JSON.parse(JSON.stringify(value)) as T); +} + +function mockKV() { + const store = new Map>(); + let gate: { + scope: string; + key: string; + release: () => void; + promise: Promise; + } | null = null; + + return { + store, + armGetGate: (scope: string, key: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + gate = { scope, key, release, promise }; + return release; + }, + get: async (scope: string, key: string): Promise => { + const value = clone((store.get(scope)?.get(key) as T) ?? null); + if (gate && gate.scope === scope && gate.key === key) { + const g = gate; + gate = null; + await g.promise; + } + return value; + }, + set: async (scope: string, key: string, data: T): Promise => { + const stored = clone(data); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, stored); + return clone(stored); + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const m = store.get(scope); + return m ? clone(Array.from(m.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const fns = new Map Promise>(); + return { + registerFunction: (id: string, handler: (data: unknown) => Promise) => { + fns.set(id, handler); + }, + trigger: (id: string, data: unknown) => fns.get(id)!(data), + }; +} + +async function flush() { + await new Promise((r) => setTimeout(r, 0)); +} + +describe("graph:persist serializes cross-path graph mutations", () => { + it("does not lose cascade's stale flag or persistGraphDelta's merge when they race on the same node", async () => { + const kv = mockKV(); + + const node: GraphNode = { + id: "node_1", + type: "concept", + name: "react", + properties: {}, + sourceObservationIds: ["obs_old"], + createdAt: "2026-03-01T00:00:00Z", + }; + await kv.set(KV.graphNodes, "node_1", node); + await kv.set(KV.graphNameIndex, "concept|react", "node_1"); + await kv.set(KV.graphNodeDegree, "node_1", 0); + await kv.set(KV.graphSnapshot, "current", { + version: 1, + topNodes: [node], + topEdges: [], + topDegrees: {}, + stats: { + totalNodes: 1, + totalEdges: 0, + nodesByType: { concept: 1 }, + edgesByType: {}, + }, + updatedAt: new Date().toISOString(), + dirty: false, + } satisfies GraphSnapshot); + + const supersededMemory: Memory = { + id: "mem_old", + createdAt: "2026-03-01T00:00:00Z", + updatedAt: "2026-03-01T00:00:00Z", + type: "fact", + title: "Old fact", + content: "Old content", + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: false, + sourceObservationIds: ["obs_old"], + }; + await kv.set(KV.memories, "mem_old", supersededMemory); + + const sdk = mockSdk(); + registerCascadeFunction(sdk as never, kv as never); + + const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, "concept|react"); + + const incomingNode: GraphNode = { + id: "node_new", + type: "concept", + name: "react", + properties: {}, + sourceObservationIds: ["obs_new"], + createdAt: new Date().toISOString(), + }; + const persistPromise = persistGraphDelta( + kv as never, + [incomingNode], + [], + ["obs_new"], + ); + + await flush(); + + const cascadePromise = sdk.trigger("mem::cascade-update", { + supersededMemoryId: "mem_old", + }) as Promise<{ success: boolean; flagged: { nodes: number } }>; + + await flush(); + releaseNameIndexRead(); + + const [, cascadeResult] = await Promise.all([persistPromise, cascadePromise]); + + expect(cascadeResult.success).toBe(true); + expect(cascadeResult.flagged.nodes).toBe(1); + + const finalNode = await kv.get(KV.graphNodes, "node_1"); + expect(finalNode!.stale).toBe(true); + expect(finalNode!.sourceObservationIds.sort()).toEqual(["obs_new", "obs_old"]); + }); +}); From 19fd27175c77487c9397a6fd002928e0a0be0592 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Sat, 26 Sep 2026 13:10:49 +0530 Subject: [PATCH 4/5] fix(state): close remaining session and graph lock gaps from review Session end (api::session::end and event::session::ended) wrote endedAt/status with an unlocked kv.update, so the locked commit linker could still read-modify-write the session row around it and revert the end. Both now take the obs:${sessionId} lock, the same key the commit linker and observe use, so the two orderings serialize instead of racing. Session start (api::session::start and event::session::started) built a fresh row and blind-set it under the lock, so if observe's implicit create had already landed, start would reset observationCount to zero and drop firstPrompt/summary/commitShas. Both now read the existing row inside the lock and merge onto it, keeping observationCount, commitShas, and firstPrompt/summary when the incoming request does not explicitly supply them. Graph import/delete (mem::import) took the graph:persist lock once per item inside the 20-wide chunked loop, which serialized the four graph node/edge import and delete loops down to one operation at a time. Each of the four loops now takes graph:persist once for the whole loop, restoring the chunk's intended concurrency. mem::graph-reset and mem::graph-snapshot-rebuild wrote the graph snapshot without the graph:persist lock, so a reset could race a concurrent extract's read-modify-write of the same snapshot and lose resetAt (breaking the post-reset orphan check) or a rebuilt count. Both writes now take graph:persist. The graph-cross-path-locks regression test gated the name-index read inside persistGraphDelta, which lets cascade's write land and then be silently re-merged back to the correct value, so the test passed even without the fix it was meant to guard. Moved the gate to the node-row read, the point the lost update actually depends on; verified it now fails on origin/main and passes on this branch. Added a session-kv-races test for api::session::commit racing api::session::end, and a sequential test for api::session::start running after mem::observe's implicit create, covering the merge behavior above. --- src/functions/export-import.ts | 26 ++++++----- src/functions/graph.ts | 8 +++- src/triggers/api.ts | 27 +++++++---- src/triggers/events.ts | 27 +++++++---- test/graph-cross-path-locks.test.ts | 4 +- test/session-kv-races.test.ts | 69 +++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 32 deletions(-) diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 9badf6003a..747cdd5307 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -373,15 +373,17 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void await kv.list(KV.insights).catch(() => []), (i) => kv.delete(KV.insights, i.id), ); - await runChunked( - await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), - (n) => - withKeyedLock("graph:persist", () => kv.delete(KV.graphNodes, n.id)), + await withKeyedLock("graph:persist", async () => + runChunked( + await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), + (n) => kv.delete(KV.graphNodes, n.id), + ), ); - await runChunked( - await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), - (e) => - withKeyedLock("graph:persist", () => kv.delete(KV.graphEdges, e.id)), + await withKeyedLock("graph:persist", async () => + runChunked( + await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), + (e) => kv.delete(KV.graphEdges, e.id), + ), ); await runChunked( await kv.list<{ id: string }>(KV.semantic).catch(() => []), @@ -478,8 +480,8 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void }); if (importData.graphNodes) { - await runChunked(importData.graphNodes, (node) => - withKeyedLock("graph:persist", async () => { + await withKeyedLock("graph:persist", () => + runChunked(importData.graphNodes!, async (node) => { if (strategy === "skip") { const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); if (existing) { stats.skipped++; return; } @@ -489,8 +491,8 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void ); } if (importData.graphEdges) { - await runChunked(importData.graphEdges, (edge) => - withKeyedLock("graph:persist", async () => { + await withKeyedLock("graph:persist", () => + runChunked(importData.graphEdges!, async (edge) => { if (strategy === "skip") { const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); if (existing) { stats.skipped++; return; } diff --git a/src/functions/graph.ts b/src/functions/graph.ts index ee4974106c..9a7c4d81e3 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -1080,7 +1080,9 @@ export function registerGraphFunction( } const snap = buildSnapshotFromArrays(nodes, edges); - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); + await withKeyedLock("graph:persist", () => + kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap), + ); const tookMs = Date.now() - started; logger.info("Graph snapshot rebuilt", { totalNodes: snap.stats.totalNodes, @@ -1137,7 +1139,9 @@ export function registerGraphFunction( ...emptySnapshot(), resetAt: new Date().toISOString(), }; - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot); + await withKeyedLock("graph:persist", () => + kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot), + ); const counts: Record = { [KV.graphSnapshot]: 1, }; diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 62ea99a93f..54f5a0e857 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -721,7 +721,7 @@ export function registerApiTriggers( ? body.agentId.trim().slice(0, 128) : undefined; const agentId = requestAgentId ?? getAgentId(); - const session: Session = { + const freshSession: Session = { id: sessionId, project, cwd, @@ -732,9 +732,18 @@ export function registerApiTriggers( ...(title ? { firstPrompt: title.slice(0, 200) } : {}), ...(agentId ? { agentId } : {}), }; - await withKeyedLock(`obs:${sessionId}`, () => - kv.set(KV.sessions, sessionId, session), - ); + const session = await withKeyedLock(`obs:${sessionId}`, async () => { + const existing = await kv.get(KV.sessions, sessionId); + const merged: Session = { + ...freshSession, + observationCount: existing?.observationCount ?? freshSession.observationCount, + firstPrompt: freshSession.firstPrompt ?? existing?.firstPrompt, + summary: freshSession.summary ?? existing?.summary, + commitShas: existing?.commitShas ?? freshSession.commitShas, + }; + await kv.set(KV.sessions, sessionId, merged); + return merged; + }); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } @@ -767,10 +776,12 @@ export function registerApiTriggers( body: { error: "sessionId is required and must be a non-empty string" }, }; } - await kv.update(KV.sessions, sessionId, [ - { type: "set", path: "endedAt", value: new Date().toISOString() }, - { type: "set", path: "status", value: "completed" }, - ]); + await withKeyedLock(`obs:${sessionId}`, () => + kv.update(KV.sessions, sessionId, [ + { type: "set", path: "endedAt", value: new Date().toISOString() }, + { type: "set", path: "status", value: "completed" }, + ]), + ); // Fan out session-stopped lifecycle (non-blocking). try { sdk.trigger({ diff --git a/src/triggers/events.ts b/src/triggers/events.ts index a8d8895da7..c3187ce5d4 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -57,7 +57,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { ? data.agentId.trim().slice(0, 128) : undefined; const agentId = requestAgentId ?? getAgentId(); - const session: Session = { + const freshSession: Session = { id: data.sessionId, project: data.project, cwd: data.cwd, @@ -66,9 +66,18 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { observationCount: 0, ...(agentId ? { agentId } : {}), }; - await withKeyedLock(`obs:${data.sessionId}`, () => - kv.set(KV.sessions, data.sessionId, session), - ); + const session = await withKeyedLock(`obs:${data.sessionId}`, async () => { + const existing = await kv.get(KV.sessions, data.sessionId); + const merged: Session = { + ...freshSession, + observationCount: existing?.observationCount ?? freshSession.observationCount, + firstPrompt: existing?.firstPrompt, + summary: existing?.summary, + commitShas: existing?.commitShas, + }; + await kv.set(KV.sessions, data.sessionId, merged); + return merged; + }); const contextResult = await sdk.trigger< { sessionId: string; project: string; agentId?: string }, { context: string } @@ -163,10 +172,12 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction( "event::session::ended", async (data: { sessionId: string }) => { - await kv.update(KV.sessions, data.sessionId, [ - { type: "set", path: "endedAt", value: new Date().toISOString() }, - { type: "set", path: "status", value: "completed" }, - ]); + await withKeyedLock(`obs:${data.sessionId}`, () => + kv.update(KV.sessions, data.sessionId, [ + { type: "set", path: "endedAt", value: new Date().toISOString() }, + { type: "set", path: "status", value: "completed" }, + ]), + ); return { success: true }; }, ); diff --git a/test/graph-cross-path-locks.test.ts b/test/graph-cross-path-locks.test.ts index c289506030..cbab748db2 100644 --- a/test/graph-cross-path-locks.test.ts +++ b/test/graph-cross-path-locks.test.ts @@ -123,7 +123,7 @@ describe("graph:persist serializes cross-path graph mutations", () => { const sdk = mockSdk(); registerCascadeFunction(sdk as never, kv as never); - const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, "concept|react"); + const releaseNodeRowRead = kv.armGetGate(KV.graphNodes, "node_1"); const incomingNode: GraphNode = { id: "node_new", @@ -147,7 +147,7 @@ describe("graph:persist serializes cross-path graph mutations", () => { }) as Promise<{ success: boolean; flagged: { nodes: number } }>; await flush(); - releaseNameIndexRead(); + releaseNodeRowRead(); const [, cascadeResult] = await Promise.all([persistPromise, cascadePromise]); diff --git a/test/session-kv-races.test.ts b/test/session-kv-races.test.ts index dbb7614acb..40c2b0f10c 100644 --- a/test/session-kv-races.test.ts +++ b/test/session-kv-races.test.ts @@ -223,4 +223,73 @@ describe("KV.sessions concurrent mutation races (finding 3)", () => { expect(finalSession).toBeTruthy(); expect(finalSession.observationCount).toBe(1); }); + + it("api::session::commit does not revert a concurrent api::session::end status update", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never); + + const session: Session = { + id: "ses_race5", + project: "/repo", + cwd: "/repo", + startedAt: new Date().toISOString(), + status: "active", + observationCount: 3, + }; + await kv.set(KV.sessions, "ses_race5", session); + + const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race5"); + + const commitPromise = sdk.trigger("api::session::commit", { + body: { sha: "deadbeef01", sessionId: "ses_race5" }, + }); + + await flush(); + + const endPromise = sdk.trigger("api::session::end", { + body: { sessionId: "ses_race5" }, + }); + + await flush(); + releaseCommitRead(); + + await Promise.all([commitPromise, endPromise]); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race5") as Session; + expect(finalSession.commitShas).toEqual(["deadbeef01"]); + expect(finalSession.status).toBe("completed"); + expect(finalSession.endedAt).toBeTruthy(); + }); + + it("api::session::start does not reset an observation count or firstPrompt already created by mem::observe", async () => { + const kv = mockKV(); + const sdk = mockSdk(); + registerObserveFunction(sdk as never, kv as never); + registerApiTriggers(sdk as never, kv as never); + sdk.fns.set("mem::context", async () => ({ context: "" })); + + await sdk.trigger("mem::observe", { + sessionId: "ses_race4", + project: "/repo", + cwd: "/repo", + hookType: "prompt_submit", + timestamp: new Date().toISOString(), + data: { prompt: "do the thing" }, + }); + + await sdk.trigger("api::session::start", { + body: { + sessionId: "ses_race4", + project: "/repo", + cwd: "/repo", + title: "fix the parser", + }, + }); + + const finalSession = kv.store.get(KV.sessions)?.get("ses_race4") as Session; + expect(finalSession.observationCount).toBe(1); + expect(finalSession.firstPrompt).toBe("fix the parser"); + expect(finalSession.status).toBe("active"); + }); }); From 263c7ba8a0b0bb8deda3fa079936ee3cbc5cb5d9 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 28 Sep 2026 11:12:55 +0530 Subject: [PATCH 5/5] fix(graph): hold graph:persist for the whole snapshot rebuild mem::graph-snapshot-rebuild listed nodes and edges and backfilled the name, edge-key and degree indexes outside the lock, and took graph:persist only for the final snapshot write. A delta that committed between the listing and that write was overwritten by a snapshot built from the older lists, and the backfill could reset degree counters the delta had just raised. The listing, index backfill and snapshot write now run in one graph:persist critical section, so an extract either lands before the rebuild lists or waits until it finishes. Nothing inside the section takes the lock again. The new concurrency test pauses the rebuild's node listing, commits a delta meanwhile, and checks that the final snapshot holds both nodes. It fails without this change. --- src/functions/graph.ts | 106 +++++++++++++------------ test/graph-extract-concurrency.test.ts | 56 ++++++++++++- 2 files changed, 108 insertions(+), 54 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 9a7c4d81e3..081887318a 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -1017,72 +1017,74 @@ export function registerGraphFunction( } try { - const [nodes, edges] = await withTimeout( - Promise.all([ - kv.list(KV.graphNodes), - kv.list(KV.graphEdges), - ]), - LIVE_ENUMERATION_BUDGET_MS, - "graph-snapshot-rebuild enumeration", - ); + const outcome = await withKeyedLock("graph:persist", async () => { + const [nodes, edges] = await withTimeout( + Promise.all([ + kv.list(KV.graphNodes), + kv.list(KV.graphEdges), + ]), + LIVE_ENUMERATION_BUDGET_MS, + "graph-snapshot-rebuild enumeration", + ); + + if (nodes.length > REBUILD_SAFE_NODE_CEILING) { + return { tooLargeNodeCount: nodes.length }; + } + + const liveNodes = nodes.filter((n) => !n.stale); + const liveEdges = edges.filter((e) => !e.stale); + const degree = new Map(); + for (const e of liveEdges) { + degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1); + degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1); + } + const BATCH_SIZE = 100; + for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) { + const batch = liveNodes.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.flatMap((n) => [ + kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id), + kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0), + ]), + ); + } + for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) { + const batch = liveEdges.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map((e) => + kv.set( + KV.graphEdgeKey, + edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type), + e.id, + ), + ), + ); + } + + const rebuilt = buildSnapshotFromArrays(nodes, edges); + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, rebuilt); + return { snap: rebuilt }; + }); - if (nodes.length > REBUILD_SAFE_NODE_CEILING) { + if ("tooLargeNodeCount" in outcome) { + const totalNodes = outcome.tooLargeNodeCount; logger.warn("Graph snapshot rebuild aborted: corpus too large", { - totalNodes: nodes.length, + totalNodes, ceiling: REBUILD_SAFE_NODE_CEILING, }); return { success: false, tooLarge: true, - totalNodes: nodes.length, + totalNodes, ceiling: REBUILD_SAFE_NODE_CEILING, error: - `Corpus has ${nodes.length} graph nodes; safe-rebuild ceiling ` + + `Corpus has ${totalNodes} graph nodes; safe-rebuild ceiling ` + `is ${REBUILD_SAFE_NODE_CEILING}. Run POST /agentmemory/graph/reset ` + `to wipe and let future extracts rebuild incrementally.`, }; } - // Backfill the targeted-lookup indexes so post-rebuild - // graph-extract calls hit the O(1) path instead of falling - // through to the (already-removed) full-scope scan. Batch - // writes via Promise.all to avoid N sequential round-trips — - // BATCH_SIZE bounds in-flight writes so we don't open thousands - // of concurrent state channels on huge corpora. - const liveNodes = nodes.filter((n) => !n.stale); - const liveEdges = edges.filter((e) => !e.stale); - const degree = new Map(); - for (const e of liveEdges) { - degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1); - degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1); - } - const BATCH_SIZE = 100; - for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) { - const batch = liveNodes.slice(i, i + BATCH_SIZE); - await Promise.all( - batch.flatMap((n) => [ - kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id), - kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0), - ]), - ); - } - for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) { - const batch = liveEdges.slice(i, i + BATCH_SIZE); - await Promise.all( - batch.map((e) => - kv.set( - KV.graphEdgeKey, - edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type), - e.id, - ), - ), - ); - } - - const snap = buildSnapshotFromArrays(nodes, edges); - await withKeyedLock("graph:persist", () => - kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap), - ); + const { snap } = outcome; const tookMs = Date.now() - started; logger.info("Graph snapshot rebuilt", { totalNodes: snap.stats.totalNodes, diff --git a/test/graph-extract-concurrency.test.ts b/test/graph-extract-concurrency.test.ts index ef3ffb4893..ca022a66a1 100644 --- a/test/graph-extract-concurrency.test.ts +++ b/test/graph-extract-concurrency.test.ts @@ -4,7 +4,7 @@ vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); -import { persistGraphDelta } from "../src/functions/graph.js"; +import { persistGraphDelta, registerGraphFunction } from "../src/functions/graph.js"; import { KV } from "../src/state/schema.js"; import type { GraphNode, GraphEdge, GraphSnapshot } from "../src/types.js"; @@ -22,9 +22,18 @@ function mockKV() { release: () => void; promise: Promise; } | null = null; + let listGate: { scope: string; release: () => void; promise: Promise } | null = null; return { store, + armListGate: (scope: string) => { + let release!: () => void; + const promise = new Promise((res) => { + release = res; + }); + listGate = { scope, release, promise }; + return release; + }, armGetGate: (scope: string, key: string) => { let release!: () => void; const promise = new Promise((res) => { @@ -53,7 +62,13 @@ function mockKV() { }, list: async (scope: string): Promise => { const m = store.get(scope); - return m ? clone(Array.from(m.values()) as T[]) : []; + const values = m ? clone(Array.from(m.values()) as T[]) : []; + if (listGate && listGate.scope === scope) { + const g = listGate; + listGate = null; + await g.promise; + } + return values; }, }; } @@ -170,4 +185,41 @@ describe("persistGraphDelta concurrent merges (finding 7)", () => { "obs_y", ]); }); + + it("snapshot rebuild does not overwrite a delta that commits while it enumerates", async () => { + const kv = mockKV(); + const handlers = new Map Promise>(); + const sdk = { + registerFunction: (id: string, fn: (data?: unknown) => Promise) => { + handlers.set(id, fn); + }, + }; + registerGraphFunction(sdk as never, kv as never, { name: "noop" } as never); + + const seeded = node({ id: "node_seed", name: "src/seed.ts" }); + await persistGraphDelta(kv as never, [seeded], [], ["obs_seed"]); + + const releaseNodeList = kv.armListGate(KV.graphNodes); + const rebuild = handlers.get("mem::graph-snapshot-rebuild")!(); + await flush(); + + const fresh = node({ id: "node_fresh", name: "src/fresh.ts" }); + const delta = persistGraphDelta(kv as never, [fresh], [], ["obs_fresh"]); + await flush(); + releaseNodeList(); + + const rebuilt = (await rebuild) as { success: boolean }; + await delta; + + expect(rebuilt.success).toBe(true); + const snapshot = (await kv.get( + KV.graphSnapshot, + "current", + )) as GraphSnapshot; + expect(snapshot.stats.totalNodes).toBe(2); + expect(snapshot.topNodes.map((n) => n.id).sort()).toEqual([ + "node_fresh", + "node_seed", + ]); + }); });