diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 626f4ff1..ae6aea88 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -5,6 +5,8 @@ import type { RawObservation, SessionSummary, Memory, + GraphNode, + GraphEdge, } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; @@ -34,6 +36,9 @@ interface EvictionStats { capEvictions: number; expiredMemories: number; nonLatestMemories: number; + staleGraphNodes?: number; + staleGraphEdges?: number; + isolatedNodes?: number; dryRun: boolean; } @@ -340,6 +345,94 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void { } } + // === Fix 1: Stale graph node/edge garbage collection === + // Single KV scan — derive stale and all lists from the same result + const allNodes = await kv.list(KV.graphNodes); + const allEdges = await kv.list(KV.graphEdges); + const staleNodes = allNodes.filter((n) => n.stale); + const staleEdges = allEdges.filter((e) => e.stale); + + for (const node of staleNodes) { + if (stats.staleGraphNodes === undefined) stats.staleGraphNodes = 0; + if (dryRun) { + stats.staleGraphNodes++; + continue; + } + try { + await kv.delete(KV.graphNodes, node.id); + stats.staleGraphNodes++; + await recordAudit(kv, "delete", "mem::evict", [node.id], { + resource: "graph-node", + reason: "stale", + dryRun, + }); + } catch (err) { + logger.warn("Eviction stale node delete failed", { + nodeId: node.id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + for (const edge of staleEdges) { + if (stats.staleGraphEdges === undefined) stats.staleGraphEdges = 0; + if (dryRun) { + stats.staleGraphEdges++; + continue; + } + try { + await kv.delete(KV.graphEdges, edge.id); + stats.staleGraphEdges++; + await recordAudit(kv, "delete", "mem::evict", [edge.id], { + resource: "graph-edge", + reason: "stale", + dryRun, + }); + } catch (err) { + logger.warn("Eviction stale edge delete failed", { + edgeId: edge.id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // === Fix 3: Isolated node cleanup (0 edges, no observations, >7 days old) === + // Reuses allNodes/allEdges from Fix 1 above + const nowTime = Date.now(); + const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + + for (const node of allNodes) { + if (node.stale) continue; // handled above + const hasEdges = allEdges.some( + (e) => e.sourceNodeId === node.id || e.targetNodeId === node.id, + ); + if (hasEdges) continue; + const hasObs = (node.sourceObservationIds ?? []).length > 0; + if (hasObs) continue; + const createdAtMs = node.createdAt ? new Date(node.createdAt).getTime() : NaN; + if (!Number.isFinite(createdAtMs)) continue; // unknown age — preserve + if (nowTime - createdAtMs < SEVEN_DAYS_MS) continue; // keep recent nodes + if (dryRun) { + if (stats.isolatedNodes === undefined) stats.isolatedNodes = 0; + stats.isolatedNodes++; + } else { + try { + await kv.delete(KV.graphNodes, node.id); + if (stats.isolatedNodes === undefined) stats.isolatedNodes = 0; + stats.isolatedNodes++; + await recordAudit(kv, "delete", "mem::evict", [node.id], { + resource: "graph-node", + reason: "isolated_no_edges_no_observations", + dryRun, + }); + } catch (err) { + logger.warn("Eviction isolated node delete failed", { + nodeId: node.id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + logger.info("Eviction complete", { stats }); return stats; }, diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 9493674d..a7d63c2b 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -7,7 +7,7 @@ import type { CompressedObservation, MemoryProvider, } from "../types.js"; -import { KV, generateId } from "../state/schema.js"; +import { KV, generateId, jaccardSimilarity } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { GRAPH_EXTRACTION_SYSTEM, @@ -399,6 +399,9 @@ function parseGraphXml( const type = attrs["type"] as GraphNode["type"] | undefined; const name = attrs["name"]; if (!type || !name) return; + const VALID_TYPES = new Set(["file", "function", "concept", "error", "decision", "pattern", "library", "person"]); + // Must match types listed in GRAPH_EXTRACTION_SYSTEM prompt — update both together + if (!VALID_TYPES.has(type)) return; // skip corrupt/invalid types from LLM const properties: Record = {}; const propRegex = /([^<]*)<\/property>/g; let propMatch; @@ -490,6 +493,9 @@ export function registerGraphFunction( let newNodeCount = 0; let newEdgeCount = 0; const newEdgesForTopCheck: GraphEdge[] = []; + // #813: cache all nodes once for fuzzy dedup, avoiding per-node kv.list calls + const allNodes = await kv.list(KV.graphNodes); + const nodeIdMap = new Map(); for (const node of nodes) { const indexKey = nameIndexKey(node.type, node.name); @@ -501,8 +507,20 @@ export function registerGraphFunction( let existing: GraphNode | null = null; if (existingId) { existing = await kv.get(KV.graphNodes, existingId); + // #813: skip stale nodes — they shouldn't merge + if (existing?.stale) existing = null; } + // #813: Fuzzy dedup via Jaccard similarity (if exact match failed) + if (!existing && node.name.length >= 3) { + existing = allNodes.find( + (n) => + !n.stale && + n.type === node.type && + n.name.length >= 3 && + jaccardSimilarity(n.name.toLowerCase(), node.name.toLowerCase()) > 0.8, + ) ?? null; + } if (existing) { const merged = mergeNode(existing, node, obsIds, capturedAt); await kv.set(KV.graphNodes, existing.id, merged); @@ -512,14 +530,19 @@ export function registerGraphFunction( (n) => n.id === existing!.id, ); if (topIdx !== -1) snap.topNodes[topIdx] = merged; + // #813: track mapping from parsed ID -> persisted ID + nodeIdMap.set(node.id, existing.id); } else { - await kv.set(KV.graphNodes, node.id, node); - await kv.set(KV.graphNameIndex, indexKey, node.id); - await kv.set(KV.graphNodeDegree, node.id, 0); + await Promise.all([ + kv.set(KV.graphNodes, node.id, node), + kv.set(KV.graphNameIndex, indexKey, node.id), + kv.set(KV.graphNodeDegree, node.id, 0), + ]); snap.stats.totalNodes += 1; snap.stats.nodesByType[node.type] = (snap.stats.nodesByType[node.type] ?? 0) + 1; newNodeCount += 1; + nodeIdMap.set(node.id, node.id); if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { // Degree 0 still beats an empty slot — sit at the tail // until edges arrive and promote. @@ -530,6 +553,20 @@ export function registerGraphFunction( } for (const edge of edges) { + // #813: remap transient parsed node IDs to persisted node IDs + const srcId = nodeIdMap.get(edge.sourceNodeId); + const tgtId = nodeIdMap.get(edge.targetNodeId); + if (!srcId || !tgtId) { + logger.warn("Edge references unknown node, skipping", { + edgeId: edge.id, + sourceNodeId: edge.sourceNodeId, + targetNodeId: edge.targetNodeId, + }); + continue; + } + edge.sourceNodeId = srcId; + edge.targetNodeId = tgtId; + const eKey = edgeIndexKey( edge.sourceNodeId, edge.targetNodeId, @@ -551,8 +588,10 @@ export function registerGraphFunction( ); if (topIdx !== -1) snap.topEdges[topIdx] = merged; } else { - await kv.set(KV.graphEdges, edge.id, edge); - await kv.set(KV.graphEdgeKey, eKey, edge.id); + await Promise.all([ + kv.set(KV.graphEdges, edge.id, edge), + kv.set(KV.graphEdgeKey, eKey, edge.id), + ]); snap.stats.totalEdges += 1; snap.stats.edgesByType[edge.type] = (snap.stats.edgesByType[edge.type] ?? 0) + 1; diff --git a/src/triggers/events.ts b/src/triggers/events.ts index e38b58db..87bb75c7 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -7,6 +7,7 @@ import { isGraphExtractionEnabled } from "../config.js"; import { logger } from "../logger.js"; export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { + const extractionLocks = new Set(); sdk.registerFunction( "event::session::started", async (data: { sessionId: string; project: string; cwd: string }) => { @@ -61,23 +62,34 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { } } if (isGraphExtractionEnabled()) { - try { - const observations = await kv.list( - KV.observations(data.sessionId), - ); - const compressed = observations.filter((o) => o.title); - if (compressed.length > 0) { - sdk.trigger({ - function_id: "mem::graph-extract", - payload: { observations: compressed }, - action: TriggerAction.Void(), + // In-process dedup guard: prevents redundant graph-extract triggers + // for the same session. Single-process only — distributed deployments + // rely on graph-extract's own idempotency. + const lockKey = `graph-extract-${data.sessionId}`; + if (extractionLocks.has(lockKey)) { + logger.info("Skipping duplicate graph-extract (lock held)", { sessionId: data.sessionId }); + } else { + extractionLocks.add(lockKey); + try { + const observations = await kv.list( + KV.observations(data.sessionId), + ); + const compressed = observations.filter((o) => o.title); + if (compressed.length > 0) { + await sdk.trigger({ + function_id: "mem::graph-extract", + payload: { observations: compressed }, + action: TriggerAction.Void(), + }); + } + } catch (err) { + logger.warn("graph-extract trigger failed", { + sessionId: data.sessionId, + error: err instanceof Error ? err.message : String(err), }); + } finally { + extractionLocks.delete(lockKey); } - } catch (err) { - logger.warn("graph-extract trigger failed", { - sessionId: data.sessionId, - error: err instanceof Error ? err.message : String(err), - }); } } return summary; diff --git a/test/extraction-locking.test.ts b/test/extraction-locking.test.ts new file mode 100644 index 00000000..41123f66 --- /dev/null +++ b/test/extraction-locking.test.ts @@ -0,0 +1,323 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/config.js", () => ({ + isGraphExtractionEnabled: () => true, +})); + +vi.mock("../src/functions/slots.js", () => ({ + isReflectEnabled: () => false, +})); + +vi.mock("iii-sdk", () => ({ + TriggerAction: { Void: () => ({}) }, +})); + +import { registerEventTriggers } from "../src/triggers/events.js"; +import { KV } from "../src/state/schema.js"; +import type { CompressedObservation } from "../src/types.js"; + +type Handler = (payload: unknown) => unknown | Promise; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + update: async ( + scope: string, + key: string, + _ops: unknown[], + ): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const handlers = new Map(); + const calls: Array<{ function_id: string; payload: unknown }> = []; + return { + calls, + sdk: { + registerFunction: (functionId: string, handler: Handler) => { + handlers.set(functionId, handler); + }, + registerTrigger: vi.fn(), + trigger: async (input: { + function_id: string; + payload: unknown; + action?: unknown; + }) => { + calls.push({ + function_id: input.function_id, + payload: input.payload, + }); + const handler = handlers.get(input.function_id); + if (!handler) + throw new Error(`missing handler: ${input.function_id}`); + return handler(input.payload); + }, + }, + }; +} + +function seedObservation( + kv: ReturnType, + sessionId: string, + obsId: string, +): void { + const obs: CompressedObservation = { + id: obsId, + sessionId, + timestamp: new Date().toISOString(), + type: "file_edit", + title: "Test observation", + facts: ["test"], + narrative: "Test narrative", + concepts: ["test"], + files: ["test.ts"], + importance: 5, + }; + kv.set(KV.observations(sessionId), obsId, obs); +} + +describe("Extraction locking in events.ts (Fix 6)", () => { + let sdk: ReturnType["sdk"]; + let calls: Array<{ function_id: string; payload: unknown }>; + let kv: ReturnType; + + beforeEach(() => { + const mock = mockSdk(); + sdk = mock.sdk; + calls = mock.calls; + kv = mockKV(); + + // Register the event handlers. + registerEventTriggers(sdk as never, kv as never); + + // Pre-register handlers that event::session::stopped calls. + sdk.registerFunction("mem::summarize", async () => ({ + summary: "test summary", + })); + sdk.registerFunction("mem::graph-extract", async () => ({ + success: true, + nodesAdded: 0, + edgesAdded: 0, + })); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("1. first extraction for a session proceeds", async () => { + seedObservation(kv, "ses_1", "obs_1"); + + const result = (await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_1" }, + })) as { summary: string }; + + // The handler returns the summary from mem::summarize. + expect(result.summary).toBe("test summary"); + + // mem::graph-extract should have been triggered. + const graphCalls = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls.length).toBe(1); + expect(graphCalls[0].payload).toEqual({ + observations: [expect.objectContaining({ id: "obs_1" })], + }); + }); + + it("2. concurrent extraction for the same session is skipped", async () => { + seedObservation(kv, "ses_1", "obs_1"); + + // Replace kv.list with a version that blocks on first call. + // This lets us interleave two handler invocations so the second + // arrives while the first holds the lock. + let listResolve: () => void; + const listBlocked = new Promise((resolve) => { + listResolve = resolve; + }); + let listCalls = 0; + const originalList = kv.list; + kv.list = async (scope: string): Promise => { + listCalls++; + if (listCalls === 1) { + // Block the first call's list — the handler has the lock here. + await listBlocked; + } + return originalList.call(kv, scope) as Promise; + }; + + // Start first call — it will run synchronously past the lock + // acquisition and then block on kv.list. + const p1 = sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_1" }, + }); + + // Yield to the event loop so the handler can reach kv.list + // (it awaits mem::summarize which resolves in a microtask). + await new Promise((r) => setTimeout(r, 5)); + expect(listCalls).toBe(1); // first call reached kv.list + + // Second call should return immediately (lock held by first). + const r2 = (await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_1" }, + })) as { summary: string }; + expect(r2.summary).toBe("test summary"); + + // No graph-extract yet — first call is still blocked. + let graphCalls = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls.length).toBe(0); + + // Unblock the first call so it can complete. + listResolve!(); + const r1 = (await p1) as { summary: string }; + expect(r1.summary).toBe("test summary"); + + // Now graph-extract should have been called exactly once. + graphCalls = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls.length).toBe(1); + }); + + it("3. after extraction completes, the lock is released (next extraction proceeds)", async () => { + seedObservation(kv, "ses_1", "obs_1"); + + // First call: acquires lock, extracts, releases. + await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_1" }, + }); + + // Second call: lock is free, should extract again. + await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_1" }, + }); + + const graphCalls = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls.length).toBe(2); + }); + + it("4. concurrent extractions for DIFFERENT sessions both proceed", async () => { + seedObservation(kv, "ses_a", "obs_a"); + seedObservation(kv, "ses_b", "obs_b"); + + // Block kv.list so we can verify both calls proceed past the lock. + let listResolve: () => void; + const listBlocked = new Promise((resolve) => { + listResolve = resolve; + }); + let listCalls = 0; + const originalList = kv.list; + kv.list = async (scope: string): Promise => { + listCalls++; + if (listCalls <= 2) { + // Block the list for both sessions (both should have acquired + // their respective locks since the lock key includes sessionId). + await listBlocked; + } + return originalList.call(kv, scope) as Promise; + }; + + // Fire both calls. Since lock keys differ (ses_a vs ses_b), both + // should acquire their locks and proceed to kv.list. + const p1 = sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_a" }, + }); + const p2 = sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_b" }, + }); + + // Yield so both handlers can reach kv.list. + await new Promise((r) => setTimeout(r, 5)); + expect(listCalls).toBe(2); // both reached kv.list + + // Unblock both. + listResolve!(); + const [r1, r2] = (await Promise.all([p1, p2])) as [ + { summary: string }, + { summary: string }, + ]; + expect(r1.summary).toBe("test summary"); + expect(r2.summary).toBe("test summary"); + + // Both should have triggered graph-extract (different locks). + const graphCalls = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls.length).toBe(2); + }); + + it("5. error during extraction still releases the lock (try/finally)", async () => { + seedObservation(kv, "ses_err", "obs_err"); + + // Make kv.list throw on the first call. + let throwOnList = true; + const originalList = kv.list; + kv.list = async (scope: string): Promise => { + if (throwOnList) { + throwOnList = false; + throw new Error("Simulated list failure"); + } + return originalList.call(kv, scope) as Promise; + }; + + // First call: kv.list throws inside the try block. + // The finally should still release the lock. + const r1 = (await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_err" }, + })) as { summary: string }; + expect(r1.summary).toBe("test summary"); + + // No graph-extract was called (list threw before extraction). + const graphCalls1 = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls1.length).toBe(0); + + // Second call: lock should be released, so extraction proceeds. + const r2 = (await sdk.trigger({ + function_id: "event::session::stopped", + payload: { sessionId: "ses_err" }, + })) as { summary: string }; + expect(r2.summary).toBe("test summary"); + + // Now graph-extract SHOULD have been called. + const graphCalls2 = calls.filter( + (c) => c.function_id === "mem::graph-extract", + ); + expect(graphCalls2.length).toBe(1); + }); +}); diff --git a/test/graph-dedup.test.ts b/test/graph-dedup.test.ts new file mode 100644 index 00000000..bb923105 --- /dev/null +++ b/test/graph-dedup.test.ts @@ -0,0 +1,248 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerGraphFunction } from "../src/functions/graph.js"; +import type { + CompressedObservation, + GraphNode, +} from "../src/types.js"; +import { jaccardSimilarity } from "../src/state/schema.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: ( + idOrOpts: string | { id: string }, + handler: Function, + ) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown }, + data?: unknown, + ) => { + const id = + typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = + typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +const mockProvider = { + name: "test", + compress: vi.fn(), + summarize: vi.fn(), +}; + +const testObs1: CompressedObservation = { + id: "obs_dedup_1", + sessionId: "ses_dedup", + timestamp: "2026-03-01T10:00:00Z", + type: "file_edit", + title: "Edit 1", + facts: ["Fact 1"], + narrative: "Narrative 1", + concepts: ["concept-a"], + files: ["a.ts"], + importance: 5, +}; + +const testObs2: CompressedObservation = { + id: "obs_dedup_2", + sessionId: "ses_dedup", + timestamp: "2026-03-01T11:00:00Z", + type: "file_edit", + title: "Edit 2", + facts: ["Fact 2"], + narrative: "Narrative 2", + concepts: ["concept-b"], + files: ["b.ts"], + importance: 5, +}; + +describe("Jaccard dedup in graph extraction (Fix 2)", () => { + let sdk: ReturnType; + let kv: ReturnType; + + beforeEach(() => { + sdk = mockSdk(); + kv = mockKV(); + vi.clearAllMocks(); + registerGraphFunction(sdk as never, kv as never, mockProvider as never); + }); + + it("1. exact name match still works (name index hit, no fuzzy scan)", async () => { + // First extract creates a node. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + // Second extract with identical name hits name-index => no fuzzy scan. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(1); + expect(nodes[0].name).toBe("MyFeature"); + }); + + it("2. Jaccard dedup merges nodes with similar names (Jaccard > 0.8)", async () => { + // "AgentMemory" and "agentmemory" produce the same word-token after + // lowercasing (both -> {"agentmemory"}) so Jaccard = 1. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(1); + + // Vanity check: confirm the Jaccard value is indeed > 0.8. + const sim = jaccardSimilarity("AgentMemory".toLowerCase(), "agentmemory".toLowerCase()); + expect(sim).toBeGreaterThan(0.8); + }); + + it("3. Jaccard dedup does NOT merge nodes with different names (Jaccard < 0.8)", async () => { + // "agentmemory" -> {"agentmemory"} + // "knowledge base" -> {"knowledge", "base"} + // No tokens in common => Jaccard = 0. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(2); + + const sim = jaccardSimilarity("agentmemory", "knowledge base"); + expect(sim).toBeLessThan(0.8); + }); + + it("4. short names (< 3 chars) are excluded from Jaccard scan", async () => { + // "ab" has length 2 < 3 => Jaccard scan skipped entirely. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + // Even though "ab" == "ab" after lowercasing, Jaccard scan is + // skipped due to short name, so a second identical short name + // creates a duplicate via a different generated ID. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + // Name-index exact match DOES still apply (name is the same, + // so nameIndexKey is identical) => deduped to 1. + // NOTE: exact match runs BEFORE the Jaccard scan. Since the name + // is identical the name index hit merges them regardless of length. + // The "short name exclusion" only matters for non-identical names. + // Verify: the exact-match name-index works for short names too. + expect(nodes.length).toBe(1); + }); + + it("4b. short names with different casing skip Jaccard scan when exact match misses", async () => { + // "Ab" (exact match key "concept|Ab") and "ab" (key "concept|ab") + // have different name-index keys. The Jaccard scan is skipped + // because "ab".length < 3, so they land as TWO separate nodes. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(2); + }); + + it("5. same name, different type => NOT merged (Jaccard respects type match)", async () => { + // First extract creates a function node named "main". + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + // Second extract creates a concept node also named "main". + // Type differs => Jaccard scan filter `n.type === node.type` fails. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(2); + expect(nodes.filter((n) => n.type === "function").length).toBe(1); + expect(nodes.filter((n) => n.type === "concept").length).toBe(1); + }); + + it("6. merged node has combined sourceObservationIds", async () => { + // Extract "AgentMemory" with obs_1, then "agentmemory" with obs_2. + // The merged node should carry both sourceObservationIds. + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs1] }); + + mockProvider.compress.mockResolvedValueOnce(` + +`); + await sdk.trigger("mem::graph-extract", { observations: [testObs2] }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(1); + const merged = nodes[0]; + + // Both observation IDs present in sourceObservationIds. + expect(merged.sourceObservationIds).toContain("obs_dedup_1"); + expect(merged.sourceObservationIds).toContain("obs_dedup_2"); + }); +}); diff --git a/test/graph-valid-types.test.ts b/test/graph-valid-types.test.ts new file mode 100644 index 00000000..96e1bdb0 --- /dev/null +++ b/test/graph-valid-types.test.ts @@ -0,0 +1,229 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerGraphFunction } from "../src/functions/graph.js"; +import type { CompressedObservation, GraphNode } from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: ( + idOrOpts: string | { id: string }, + handler: Function, + ) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown }, + data?: unknown, + ) => { + const id = + typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = + typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +const mockProvider = { + name: "test", + compress: vi.fn(), + summarize: vi.fn(), +}; + +const testObs: CompressedObservation = { + id: "obs_vt_1", + sessionId: "ses_vt", + timestamp: "2026-04-01T10:00:00Z", + type: "file_edit", + title: "Valid types test", + facts: ["Test"], + narrative: "Testing VALID_TYPES guard", + concepts: ["testing"], + files: ["test.ts"], + importance: 3, +}; + +describe("parseGraphXml — VALID_TYPES guard (Fix 4)", () => { + let sdk: ReturnType; + let kv: ReturnType; + + beforeEach(() => { + sdk = mockSdk(); + kv = mockKV(); + vi.clearAllMocks(); + registerGraphFunction(sdk as never, kv as never, mockProvider as never); + }); + + const VALID_TYPES = [ + "file", + "function", + "concept", + "error", + "decision", + "pattern", + "library", + "person", + ]; + + for (const validType of VALID_TYPES) { + it(`1. accepts valid type "${validType}"`, async () => { + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(1); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(1); + expect(nodes[0].type).toBe(validType); + }); + } + + it("2a. rejects invalid type string", async () => { + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(0); + }); + + it("2b. rejects pipe-concatenated type string", async () => { + // LLMs sometimes emit the entire type list when confused. + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); + + it("2c. rejects empty type string", async () => { + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); + + it("3a. missing type attribute returns early (no crash, no node)", async () => { + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); + + it("3b. missing name attribute returns early (no crash, no node)", async () => { + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); + + it("4. mixed valid+invalid entities: valid ones pass, invalid are skipped", async () => { + mockProvider.compress.mockResolvedValueOnce(` + + + + + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(3); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.length).toBe(3); + expect(nodes.find((n) => n.name === "src/index.ts")).toBeDefined(); + expect(nodes.find((n) => n.name === "main")).toBeDefined(); + expect(nodes.find((n) => n.name === "MyConcept")).toBeDefined(); + expect(nodes.find((n) => n.name === "bad-one")).toBeUndefined(); + expect(nodes.find((n) => n.name === "also-bad")).toBeUndefined(); + }); + + it("5a. type with trailing whitespace is rejected", async () => { + // The VALID_TYPES set contains exact strings — "file " != "file". + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); + + it("5b. mixed-case type is rejected", async () => { + // VALID_TYPES is lowercase-only — "File" !== "file". + mockProvider.compress.mockResolvedValueOnce(` + +`); + const result = (await sdk.trigger("mem::graph-extract", { + observations: [testObs], + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + expect(result.nodesAdded).toBe(0); + }); +});