From 8dedbf37595e55c524a25a063a2ee3bc236f4aee Mon Sep 17 00:00:00 2001 From: Matt Youngberg <46580755+mattyoungberg@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:07:39 -0700 Subject: [PATCH] fix(graph): batch the LLM extraction pass by GRAPH_EXTRACTION_BATCH_SIZE mem::graph-extract sent every observation it received to the model in a single prompt. The session-stop fan-out hands it the whole session, so long sessions produce prompts far beyond the context window of small or local models: the request is rejected (counting toward the shared circuit breaker) or silently truncated, and the extraction yields zero nodes. GRAPH_EXTRACTION_BATCH_SIZE was documented and parsed by getGraphBatchSize() but never read. The LLM pass now runs one call per batch of GRAPH_EXTRACTION_BATCH_SIZE observations (default 10), sequentially. A failed batch is logged and the remaining batches still contribute; the call fails only when every batch fails and nothing was extracted. Each node and edge cites only its own batch's observations. The prompt builder caps each narrative at 1500 characters and concepts/files at 20 entries, so a batch's prompt size is predictable. This is independent of the incremental-dispatch work in #1268 / #1639, which reduces how many observations reach the function; batching bounds what one call sends no matter how many arrive (session stop, /graph/extract, /graph/build). Refs #978 Signed-off-by: Matt Youngberg <46580755+mattyoungberg@users.noreply.github.com> --- src/functions/graph.ts | 58 +++++++---- src/prompts/graph-extraction.ts | 7 +- test/graph-extract-batching.test.ts | 156 ++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 21 deletions(-) create mode 100644 test/graph-extract-batching.test.ts diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 096a4872ca..77e44bd248 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -18,7 +18,7 @@ import { GRAPH_EXTRACTION_SYSTEM, buildGraphExtractionPrompt, } from "../prompts/graph-extraction.js"; -import { isGraphExtractionEnabled } from "../config.js"; +import { getGraphBatchSize, isGraphExtractionEnabled } from "../config.js"; import { recordAudit, safeAudit } from "./audit.js"; import { logger } from "../logger.js"; import { scrubRecord } from "./privacy.js"; @@ -744,27 +744,45 @@ export function registerGraphFunction( isGraphExtractionEnabled() && !provider.name.includes("noop"); let llmError: string | undefined; if (llmEnabled) { - const prompt = buildGraphExtractionPrompt( - data.observations.map((o) => ({ - title: o.title, - narrative: o.narrative, - concepts: o.concepts, - files: o.files, - type: o.type, - })), - ); - try { - const response = await provider.compress( - GRAPH_EXTRACTION_SYSTEM, - prompt, + // One call per batch keeps each prompt inside small context windows, + // and a failed batch does not discard the others. + const batchSize = Math.max(1, getGraphBatchSize()); + let failedBatches = 0; + let batchCount = 0; + for (let i = 0; i < data.observations.length; i += batchSize) { + const batch = data.observations.slice(i, i + batchSize); + batchCount++; + const prompt = buildGraphExtractionPrompt( + batch.map((o) => ({ + title: o.title, + narrative: o.narrative, + concepts: o.concepts, + files: o.files, + type: o.type, + })), ); - const parsed = parseGraphXml(response, obsIds); - nodes = nodes.concat(parsed.nodes); - edges = edges.concat(parsed.edges); - } catch (err) { - llmError = err instanceof Error ? err.message : String(err); - logger.error("LLM graph extraction failed", { error: llmError }); + try { + const response = await provider.compress( + GRAPH_EXTRACTION_SYSTEM, + prompt, + ); + const parsed = parseGraphXml( + response, + batch.map((o) => o.id), + ); + nodes = nodes.concat(parsed.nodes); + edges = edges.concat(parsed.edges); + } catch (err) { + failedBatches++; + llmError = err instanceof Error ? err.message : String(err); + logger.error("LLM graph extraction failed", { + error: llmError, + batch: batchCount, + batchSize: batch.length, + }); + } } + if (failedBatches < batchCount) llmError = undefined; } if (nodes.length === 0 && edges.length === 0) { diff --git a/src/prompts/graph-extraction.ts b/src/prompts/graph-extraction.ts index cb6d47ad82..a6b1917e7f 100644 --- a/src/prompts/graph-extraction.ts +++ b/src/prompts/graph-extraction.ts @@ -16,6 +16,11 @@ Rules: - Weight relationships by how strong/direct the connection is - If no entities found, output empty tags`; +// Bounds per observation so a batch prompt has a predictable size. Long +// narratives come from uncompressed captures; the head carries the signal. +export const GRAPH_PROMPT_MAX_NARRATIVE_CHARS = 1500; +export const GRAPH_PROMPT_MAX_LIST_ITEMS = 20; + export function buildGraphExtractionPrompt( observations: Array<{ title: string; @@ -28,7 +33,7 @@ export function buildGraphExtractionPrompt( const items = observations .map( (o, i) => - `[${i + 1}] Type: ${o.type}\nTitle: ${o.title}\nNarrative: ${o.narrative}\nConcepts: ${(o.concepts ?? []).join(", ")}\nFiles: ${(o.files ?? []).join(", ")}`, + `[${i + 1}] Type: ${o.type}\nTitle: ${o.title}\nNarrative: ${(o.narrative ?? "").slice(0, GRAPH_PROMPT_MAX_NARRATIVE_CHARS)}\nConcepts: ${(o.concepts ?? []).slice(0, GRAPH_PROMPT_MAX_LIST_ITEMS).join(", ")}\nFiles: ${(o.files ?? []).slice(0, GRAPH_PROMPT_MAX_LIST_ITEMS).join(", ")}`, ) .join("\n\n"); // Some local models default to a hidden reasoning pass that consumes diff --git a/test/graph-extract-batching.test.ts b/test/graph-extract-batching.test.ts new file mode 100644 index 0000000000..c1a1f9f6d1 --- /dev/null +++ b/test/graph-extract-batching.test.ts @@ -0,0 +1,156 @@ +import { createGraphIndexState } from "./helpers/graph-index-state.js"; +import { initializeGraphIndex } from "../src/state/graph-retrieval-index.js"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerGraphFunction } from "../src/functions/graph.js"; +import { + GRAPH_PROMPT_MAX_LIST_ITEMS, + GRAPH_PROMPT_MAX_NARRATIVE_CHARS, + buildGraphExtractionPrompt, +} from "../src/prompts/graph-extraction.js"; +import type { CompressedObservation, GraphNode } from "../src/types.js"; + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async (id: string, payload: unknown) => { + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +function obs(n: number): CompressedObservation { + return { + id: `obs_${n}`, + sessionId: "ses_1", + timestamp: `2026-02-01T10:00:0${n}Z`, + type: "file_edit", + title: `Observation ${n}`, + facts: [], + narrative: `Narrative ${n}`, + concepts: [], + files: [], + importance: 5, + }; +} + +function entityXml(name: string): string { + return ``; +} + +function countObservations(prompt: string): number { + return (prompt.match(/^\[\d+\] Type:/gm) ?? []).length; +} + +describe("mem::graph-extract LLM batching", () => { + const ORIG = { + flag: process.env["GRAPH_EXTRACTION_ENABLED"], + batch: process.env["GRAPH_EXTRACTION_BATCH_SIZE"], + }; + let sdk: ReturnType; + let kv: ReturnType["kv"]; + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn() }; + + beforeEach(async () => { + vi.clearAllMocks(); + process.env["GRAPH_EXTRACTION_ENABLED"] = "true"; + process.env["GRAPH_EXTRACTION_BATCH_SIZE"] = "2"; + sdk = mockSdk(); + kv = createGraphIndexState().kv; + await initializeGraphIndex(kv, { status: "ready" }); + registerGraphFunction(sdk as never, kv as never, provider as never); + }); + + afterEach(() => { + for (const [key, value] of [ + ["GRAPH_EXTRACTION_ENABLED", ORIG.flag], + ["GRAPH_EXTRACTION_BATCH_SIZE", ORIG.batch], + ] as const) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + it("sends at most GRAPH_EXTRACTION_BATCH_SIZE observations per LLM call", async () => { + provider.compress.mockImplementation(async () => entityXml("fn")); + + const result = (await sdk.trigger("mem::graph-extract", { + observations: [1, 2, 3, 4, 5].map(obs), + })) as { success: boolean }; + + expect(result.success).toBe(true); + expect(provider.compress).toHaveBeenCalledTimes(3); + const sizes = provider.compress.mock.calls.map(([, prompt]) => countObservations(prompt as string)); + expect(sizes).toEqual([2, 2, 1]); + }); + + it("keeps the entities of successful batches when another batch fails", async () => { + provider.compress + .mockResolvedValueOnce(entityXml("first")) + .mockRejectedValueOnce(new Error("upstream 400")) + .mockResolvedValueOnce(entityXml("third")); + + const result = (await sdk.trigger("mem::graph-extract", { + observations: [1, 2, 3, 4, 5].map(obs), + })) as { success: boolean; nodesAdded: number }; + + expect(result.success).toBe(true); + const names = (await kv.list("mem:graph:nodes")).map((n) => n.name).sort(); + expect(names).toEqual(["first", "third"]); + }); + + it("records only the batch's own observations as a node's sources", async () => { + provider.compress + .mockResolvedValueOnce(entityXml("first")) + .mockResolvedValueOnce(entityXml("second")); + + await sdk.trigger("mem::graph-extract", { observations: [1, 2, 3].map(obs) }); + + const nodes = await kv.list("mem:graph:nodes"); + expect(nodes.find((n) => n.name === "first")?.sourceObservationIds.sort()).toEqual(["obs_1", "obs_2"]); + expect(nodes.find((n) => n.name === "second")?.sourceObservationIds).toEqual(["obs_3"]); + }); + + it("fails when every batch fails and nothing was extracted", async () => { + provider.compress.mockRejectedValue(new Error("model down")); + + const result = (await sdk.trigger("mem::graph-extract", { + observations: [1, 2, 3].map(obs), + })) as { success: boolean; error?: string }; + + expect(result.success).toBe(false); + expect(result.error).toBe("model down"); + }); +}); + +describe("buildGraphExtractionPrompt bounds", () => { + it("caps narrative length and list sizes per observation", () => { + const many = Array.from({ length: GRAPH_PROMPT_MAX_LIST_ITEMS + 5 }, (_, i) => `item${i}`); + const prompt = buildGraphExtractionPrompt([ + { + type: "command_run", + title: "Long capture", + narrative: "x".repeat(GRAPH_PROMPT_MAX_NARRATIVE_CHARS * 4), + concepts: many, + files: many, + }, + ]); + + expect(prompt).toContain("x".repeat(GRAPH_PROMPT_MAX_NARRATIVE_CHARS)); + expect(prompt).not.toContain("x".repeat(GRAPH_PROMPT_MAX_NARRATIVE_CHARS + 1)); + expect(prompt).toContain(`item${GRAPH_PROMPT_MAX_LIST_ITEMS - 1}`); + expect(prompt).not.toContain(`item${GRAPH_PROMPT_MAX_LIST_ITEMS},`); + expect(prompt).not.toContain(`item${GRAPH_PROMPT_MAX_LIST_ITEMS + 4}`); + }); +});