diff --git a/src/config.ts b/src/config.ts index a6c62ce55e..eaaa293c25 100644 --- a/src/config.ts +++ b/src/config.ts @@ -385,6 +385,26 @@ export function getGraphBatchSize(): number { return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10); } +// Upstream #1168 / #1171: `sourceObservationIds` on graph nodes (and the +// same field on edges) was capped at creation but re-unioned without a +// cap on every merge. Because extraction re-observes the same entities +// continuously, the array grew monotonically for the life of the graph +// and ended up as ~97-99% of all bytes in the collection — measured here +// at 9.55 MB of 9.4 MB across 500 nodes, with one `package.json` node +// holding 4,707 ids in 133 KB. Provenance past the most recent handful +// carries almost no signal, so bound it and keep the newest. +const GRAPH_MAX_SOURCE_IDS_DEFAULT = 10; + +export function getGraphMaxSourceIds(): number { + return Math.max( + 1, + safeParseInt( + getMergedEnv()["GRAPH_MAX_SOURCE_IDS"], + GRAPH_MAX_SOURCE_IDS_DEFAULT, + ), + ); +} + // window for the smart-search followup-rate diagnostic. A second // search arriving within this many seconds (with disjoint results) // counts as a "follow-up" — a directional signal that the first result diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 76340d60fc..19d5df9c4b 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -13,7 +13,10 @@ import { GRAPH_EXTRACTION_SYSTEM, buildGraphExtractionPrompt, } from "../prompts/graph-extraction.js"; -import { isGraphExtractionEnabled } from "../config.js"; +import { + isGraphExtractionEnabled, + getGraphMaxSourceIds, +} from "../config.js"; import { recordAudit } from "./audit.js"; import { logger } from "../logger.js"; @@ -22,6 +25,37 @@ import { logger } from "../logger.js"; // reported 11k-node / 28k-edge corpus, and 5,000 is the upper bound a // caller can request explicitly. Tuned conservatively because edges // fan out faster than nodes. +// How many provenance ids a projected graph-query response keeps as a +// sample. The full array is available with `includeSources: true`. +const GRAPH_QUERY_SOURCE_SAMPLE = 3; + +/** Bound a provenance array, keeping the most recent ids. */ +function capSourceIds(ids: string[]): string[] { + const max = getGraphMaxSourceIds(); + return ids.length <= max ? ids : ids.slice(-max); +} + +/** + * Strip the provenance array down to a count plus a small sample. + * graph-query used to return node objects verbatim, so every consumer + * paid for accumulated provenance on every call — for an agent that + * cost is context window (upstream #1171). + */ +function projectSources( + row: T, + includeSources: boolean, +): T & { sourceObservationCount?: number } { + const ids = row.sourceObservationIds ?? []; + if (includeSources || ids.length <= GRAPH_QUERY_SOURCE_SAMPLE) { + return { ...row, sourceObservationCount: ids.length }; + } + return { + ...row, + sourceObservationIds: ids.slice(-GRAPH_QUERY_SOURCE_SAMPLE), + sourceObservationCount: ids.length, + }; +} + const DEFAULT_GRAPH_QUERY_LIMIT = 500; const MAX_GRAPH_QUERY_LIMIT = 5000; @@ -146,6 +180,7 @@ function paginateFromSnapshot( filterType: string | undefined, limit: number, offset: number, + includeSources = false, ): GraphQueryResult { const filteredNodes = filterType ? snap.topNodes.filter((n) => n.type === filterType) @@ -159,8 +194,8 @@ function paginateFromSnapshot( (e) => pageIds.has(e.sourceNodeId) && pageIds.has(e.targetNodeId), ); return { - nodes: pageNodes, - edges: pageEdges, + nodes: pageNodes.map((n) => projectSources(n, includeSources)), + edges: pageEdges.map((e) => projectSources(e, includeSources)), depth: 0, totalNodes: total, totalEdges: snap.stats.totalEdges, @@ -280,13 +315,15 @@ function mergeNode( ): GraphNode { return { ...existing, - sourceObservationIds: [ + // Newest ids sort last through the Set, so the cap keeps the most + // recent provenance and drops the oldest (upstream #1171). + sourceObservationIds: capSourceIds([ ...new Set([ ...existing.sourceObservationIds, ...incoming.sourceObservationIds, ...obsIds, ]), - ], + ]), properties: { ...existing.properties, ...incoming.properties }, updatedAt: capturedAt, }; @@ -298,9 +335,9 @@ function mergeEdge( ): GraphEdge { return { ...existing, - sourceObservationIds: [ + sourceObservationIds: capSourceIds([ ...new Set([...existing.sourceObservationIds, ...obsIds]), - ], + ]), }; } @@ -327,6 +364,7 @@ function paginate( depth: number, limit: number, offset: number, + includeSources = false, ): GraphQueryResult { const totalNodes = nodes.length; const pageNodes = nodes.slice(offset, offset + limit); @@ -349,8 +387,8 @@ function paginate( 0, ); return { - nodes: pageNodes, - edges: pageEdges, + nodes: pageNodes.map((n) => projectSources(n, includeSources)), + edges: pageEdges.map((e) => projectSources(e, includeSources)), depth, totalNodes, totalEdges, @@ -411,7 +449,7 @@ function parseGraphXml( type, name, properties, - sourceObservationIds: observationIds, + sourceObservationIds: capSourceIds(observationIds), createdAt: now, }); }; @@ -443,7 +481,7 @@ function parseGraphXml( sourceNodeId: sourceNode.id, targetNodeId: targetNode.id, weight: Math.max(0, Math.min(1, weight)), - sourceObservationIds: observationIds, + sourceObservationIds: capSourceIds(observationIds), createdAt: now, }); } @@ -484,7 +522,10 @@ export function extractGraphHeuristics( nodeByKey.set(key, node); nodes.push(node); } else if (!node.sourceObservationIds.includes(obsId)) { - node.sourceObservationIds.push(obsId); + node.sourceObservationIds = capSourceIds([ + ...node.sourceObservationIds, + obsId, + ]); } return node; }; @@ -497,7 +538,10 @@ export function extractGraphHeuristics( const existing = edgeByPair.get(pair); if (existing) { if (!existing.sourceObservationIds.includes(obs.id)) { - existing.sourceObservationIds.push(obs.id); + existing.sourceObservationIds = capSourceIds([ + ...existing.sourceObservationIds, + obs.id, + ]); } return; } @@ -784,9 +828,13 @@ export function registerGraphFunction( query?: string; limit?: number; offset?: number; + includeSources?: boolean; }): Promise => { const maxDepth = Math.min(data.maxDepth || 3, 5); const { limit, offset } = resolvePagination(data.limit, data.offset); + // Off by default: the full provenance array is ~99% of the bytes + // and almost never what the caller wanted (upstream #1171). + const includeSources = data.includeSources === true; // #814 v2: the empty-body / nodeType-only path NEVER enumerates. // It reads the snapshot exclusively. The snapshot is updated @@ -799,7 +847,7 @@ export function registerGraphFunction( if (noWalk) { const snap = await readSnapshot(kv); if (snap && snap.stats.totalNodes > 0) { - return paginateFromSnapshot(snap, data.nodeType, limit, offset); + return paginateFromSnapshot(snap, data.nodeType, limit, offset, includeSources); } return { nodes: [], @@ -875,7 +923,7 @@ export function registerGraphFunction( (v) => typeof v === "string" && v.toLowerCase().includes(lower), ), ); - return paginate(matchingNodes, allEdges, 0, limit, offset); + return paginate(matchingNodes, allEdges, 0, limit, offset, includeSources); } if (data.startNodeId) { @@ -917,11 +965,11 @@ export function registerGraphFunction( } } - return paginate(resultNodes, resultEdges, maxDepth, limit, offset); + return paginate(resultNodes, resultEdges, maxDepth, limit, offset, includeSources); } // Unreachable — noWalk branch handles the rest. - return paginate([], [], 0, limit, offset); + return paginate([], [], 0, limit, offset, includeSources); }, ); diff --git a/src/types.ts b/src/types.ts index d2c63efa61..b5b786a8fb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -409,6 +409,10 @@ export interface GraphNode { updatedAt?: string; aliases?: string[]; stale?: boolean; + /** Present on graph-query responses: true length of sourceObservationIds + * before projection. The array itself is a recent sample unless the + * caller passed includeSources (upstream #1171). */ + sourceObservationCount?: number; } export type GraphEdgeType = @@ -445,6 +449,10 @@ export interface GraphEdge { supersededBy?: string; isLatest?: boolean; stale?: boolean; + /** Present on graph-query responses: true length of sourceObservationIds + * before projection. The array itself is a recent sample unless the + * caller passed includeSources (upstream #1171). */ + sourceObservationCount?: number; } export interface EdgeContext { diff --git a/test/graph-provenance-cap.test.ts b/test/graph-provenance-cap.test.ts new file mode 100644 index 0000000000..73dded2c71 --- /dev/null +++ b/test/graph-provenance-cap.test.ts @@ -0,0 +1,174 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerGraphFunction } from "../src/functions/graph.js"; +import type { + CompressedObservation, + GraphNode, + GraphQueryResult, +} from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id; + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown }, + data?: unknown, + ) => { + const id = + typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + }; +} + +// One entity, no relationships: every extract re-observes the same node, +// which is the shape that made the provenance array grow without bound. +const mockProvider = { + name: "test", + compress: vi.fn().mockResolvedValue(` +src/hot-path.ts + + +`), + summarize: vi.fn(), +}; + +function obs(n: number): CompressedObservation { + return { + id: `obs_${n}`, + sessionId: "ses_1", + timestamp: `2026-02-01T10:00:${String(n % 60).padStart(2, "0")}Z`, + type: "file_edit", + title: `Edit ${n}`, + facts: [`change ${n}`], + narrative: `Edited the hot path, revision ${n}`, + concepts: [], + files: [], + importance: 5, + }; +} + +describe("graph provenance bounds (#1168 / #1171)", () => { + let sdk: ReturnType; + let kv: ReturnType; + const ORIG_FLAG = process.env["GRAPH_EXTRACTION_ENABLED"]; + const ORIG_MAX = process.env["GRAPH_MAX_SOURCE_IDS"]; + + beforeEach(() => { + sdk = mockSdk(); + kv = mockKV(); + vi.clearAllMocks(); + process.env["GRAPH_EXTRACTION_ENABLED"] = "true"; + process.env["GRAPH_MAX_SOURCE_IDS"] = "5"; + registerGraphFunction(sdk as never, kv as never, mockProvider as never); + }); + + afterEach(() => { + if (ORIG_FLAG === undefined) delete process.env["GRAPH_EXTRACTION_ENABLED"]; + else process.env["GRAPH_EXTRACTION_ENABLED"] = ORIG_FLAG; + if (ORIG_MAX === undefined) delete process.env["GRAPH_MAX_SOURCE_IDS"]; + else process.env["GRAPH_MAX_SOURCE_IDS"] = ORIG_MAX; + }); + + // The regression: creation capped at ten ids, mergeNode re-unioned with + // no cap, so re-observing an entity grew the array for the life of the + // graph. + it("keeps sourceObservationIds bounded across repeated merges", async () => { + for (let i = 1; i <= 20; i++) { + await sdk.trigger("mem::graph-extract", { observations: [obs(i)] }); + } + + const nodes = await kv.list("mem:graph:nodes"); + const hot = nodes.find((n) => n.name === "src/hot-path.ts")!; + expect(hot).toBeDefined(); + expect(hot.sourceObservationIds.length).toBeLessThanOrEqual(5); + // Newest kept, oldest dropped. + expect(hot.sourceObservationIds).toContain("obs_20"); + expect(hot.sourceObservationIds).not.toContain("obs_1"); + }); + + it("caps provenance at creation too", async () => { + await sdk.trigger("mem::graph-extract", { + observations: [obs(1), obs(2), obs(3), obs(4), obs(5), obs(6), obs(7)], + }); + + const nodes = await kv.list("mem:graph:nodes"); + for (const node of nodes) { + expect(node.sourceObservationIds.length).toBeLessThanOrEqual(5); + } + }); + + it("graph-query projects provenance to a count plus a sample", async () => { + for (let i = 1; i <= 20; i++) { + await sdk.trigger("mem::graph-extract", { observations: [obs(i)] }); + } + + const result = (await sdk.trigger("mem::graph-query", { + query: "hot-path", + })) as GraphQueryResult; + + const node = result.nodes.find((n) => n.name === "src/hot-path.ts")!; + expect(node).toBeDefined(); + expect(node.sourceObservationCount).toBeGreaterThan(0); + expect(node.sourceObservationIds.length).toBeLessThanOrEqual(3); + }); + + it("graph-query returns the full array when includeSources is set", async () => { + for (let i = 1; i <= 20; i++) { + await sdk.trigger("mem::graph-extract", { observations: [obs(i)] }); + } + + const projected = (await sdk.trigger("mem::graph-query", { + query: "hot-path", + })) as GraphQueryResult; + const full = (await sdk.trigger("mem::graph-query", { + query: "hot-path", + includeSources: true, + })) as GraphQueryResult; + + const projectedNode = projected.nodes.find( + (n) => n.name === "src/hot-path.ts", + )!; + const fullNode = full.nodes.find((n) => n.name === "src/hot-path.ts")!; + + expect(fullNode.sourceObservationIds.length).toBe( + fullNode.sourceObservationCount, + ); + expect(fullNode.sourceObservationIds.length).toBeGreaterThanOrEqual( + projectedNode.sourceObservationIds.length, + ); + }); +});