Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,26 @@ export function getGraphBatchSize(): number {
return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10);
}

// Upstream #1168 / #1171: `sourceObservationIds` on graph nodes (and the
// same field on edges) was capped at creation but re-unioned without a
// cap on every merge. Because extraction re-observes the same entities
// continuously, the array grew monotonically for the life of the graph
// and ended up as ~97-99% of all bytes in the collection — measured here
// at 9.55 MB of 9.4 MB across 500 nodes, with one `package.json` node
// holding 4,707 ids in 133 KB. Provenance past the most recent handful
// carries almost no signal, so bound it and keep the newest.
const GRAPH_MAX_SOURCE_IDS_DEFAULT = 10;

export function getGraphMaxSourceIds(): number {
return Math.max(
1,
safeParseInt(
getMergedEnv()["GRAPH_MAX_SOURCE_IDS"],
GRAPH_MAX_SOURCE_IDS_DEFAULT,
),
);
}

// window for the smart-search followup-rate diagnostic. A second
// search arriving within this many seconds (with disjoint results)
// counts as a "follow-up" — a directional signal that the first result
Expand Down
82 changes: 65 additions & 17 deletions src/functions/graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ import {
GRAPH_EXTRACTION_SYSTEM,
buildGraphExtractionPrompt,
} from "../prompts/graph-extraction.js";
import { isGraphExtractionEnabled } from "../config.js";
import {
isGraphExtractionEnabled,
getGraphMaxSourceIds,
} from "../config.js";
import { recordAudit } from "./audit.js";
import { logger } from "../logger.js";

Expand All @@ -22,6 +25,37 @@ import { logger } from "../logger.js";
// reported 11k-node / 28k-edge corpus, and 5,000 is the upper bound a
// caller can request explicitly. Tuned conservatively because edges
// fan out faster than nodes.
// How many provenance ids a projected graph-query response keeps as a
// sample. The full array is available with `includeSources: true`.
const GRAPH_QUERY_SOURCE_SAMPLE = 3;

/** Bound a provenance array, keeping the most recent ids. */
function capSourceIds(ids: string[]): string[] {
const max = getGraphMaxSourceIds();
return ids.length <= max ? ids : ids.slice(-max);
}

/**
* Strip the provenance array down to a count plus a small sample.
* graph-query used to return node objects verbatim, so every consumer
* paid for accumulated provenance on every call — for an agent that
* cost is context window (upstream #1171).
*/
function projectSources<T extends { sourceObservationIds?: string[] }>(
row: T,
includeSources: boolean,
): T & { sourceObservationCount?: number } {
const ids = row.sourceObservationIds ?? [];
if (includeSources || ids.length <= GRAPH_QUERY_SOURCE_SAMPLE) {
return { ...row, sourceObservationCount: ids.length };
}
return {
...row,
sourceObservationIds: ids.slice(-GRAPH_QUERY_SOURCE_SAMPLE),
sourceObservationCount: ids.length,
};
}

const DEFAULT_GRAPH_QUERY_LIMIT = 500;
const MAX_GRAPH_QUERY_LIMIT = 5000;

Expand Down Expand Up @@ -146,6 +180,7 @@ function paginateFromSnapshot(
filterType: string | undefined,
limit: number,
offset: number,
includeSources = false,
): GraphQueryResult {
const filteredNodes = filterType
? snap.topNodes.filter((n) => n.type === filterType)
Expand All @@ -159,8 +194,8 @@ function paginateFromSnapshot(
(e) => pageIds.has(e.sourceNodeId) && pageIds.has(e.targetNodeId),
);
return {
nodes: pageNodes,
edges: pageEdges,
nodes: pageNodes.map((n) => projectSources(n, includeSources)),
edges: pageEdges.map((e) => projectSources(e, includeSources)),
depth: 0,
totalNodes: total,
totalEdges: snap.stats.totalEdges,
Expand Down Expand Up @@ -280,13 +315,15 @@ function mergeNode(
): GraphNode {
return {
...existing,
sourceObservationIds: [
// Newest ids sort last through the Set, so the cap keeps the most
// recent provenance and drops the oldest (upstream #1171).
sourceObservationIds: capSourceIds([
...new Set([
...existing.sourceObservationIds,
...incoming.sourceObservationIds,
...obsIds,
]),
],
]),
properties: { ...existing.properties, ...incoming.properties },
updatedAt: capturedAt,
};
Expand All @@ -298,9 +335,9 @@ function mergeEdge(
): GraphEdge {
return {
...existing,
sourceObservationIds: [
sourceObservationIds: capSourceIds([
...new Set([...existing.sourceObservationIds, ...obsIds]),
],
]),
};
}

Expand All @@ -327,6 +364,7 @@ function paginate(
depth: number,
limit: number,
offset: number,
includeSources = false,
): GraphQueryResult {
const totalNodes = nodes.length;
const pageNodes = nodes.slice(offset, offset + limit);
Expand All @@ -349,8 +387,8 @@ function paginate(
0,
);
return {
nodes: pageNodes,
edges: pageEdges,
nodes: pageNodes.map((n) => projectSources(n, includeSources)),
edges: pageEdges.map((e) => projectSources(e, includeSources)),
depth,
totalNodes,
totalEdges,
Expand Down Expand Up @@ -411,7 +449,7 @@ function parseGraphXml(
type,
name,
properties,
sourceObservationIds: observationIds,
sourceObservationIds: capSourceIds(observationIds),
createdAt: now,
});
};
Expand Down Expand Up @@ -443,7 +481,7 @@ function parseGraphXml(
sourceNodeId: sourceNode.id,
targetNodeId: targetNode.id,
weight: Math.max(0, Math.min(1, weight)),
sourceObservationIds: observationIds,
sourceObservationIds: capSourceIds(observationIds),
createdAt: now,
});
}
Expand Down Expand Up @@ -484,7 +522,10 @@ export function extractGraphHeuristics(
nodeByKey.set(key, node);
nodes.push(node);
} else if (!node.sourceObservationIds.includes(obsId)) {
node.sourceObservationIds.push(obsId);
node.sourceObservationIds = capSourceIds([
...node.sourceObservationIds,
obsId,
]);
}
return node;
};
Expand All @@ -497,7 +538,10 @@ export function extractGraphHeuristics(
const existing = edgeByPair.get(pair);
if (existing) {
if (!existing.sourceObservationIds.includes(obs.id)) {
existing.sourceObservationIds.push(obs.id);
existing.sourceObservationIds = capSourceIds([
...existing.sourceObservationIds,
obs.id,
]);
}
return;
}
Expand Down Expand Up @@ -784,9 +828,13 @@ export function registerGraphFunction(
query?: string;
limit?: number;
offset?: number;
includeSources?: boolean;
}): Promise<GraphQueryResult> => {
const maxDepth = Math.min(data.maxDepth || 3, 5);
const { limit, offset } = resolvePagination(data.limit, data.offset);
// Off by default: the full provenance array is ~99% of the bytes
// and almost never what the caller wanted (upstream #1171).
const includeSources = data.includeSources === true;

// #814 v2: the empty-body / nodeType-only path NEVER enumerates.
// It reads the snapshot exclusively. The snapshot is updated
Expand All @@ -799,7 +847,7 @@ export function registerGraphFunction(
if (noWalk) {
const snap = await readSnapshot(kv);
if (snap && snap.stats.totalNodes > 0) {
return paginateFromSnapshot(snap, data.nodeType, limit, offset);
return paginateFromSnapshot(snap, data.nodeType, limit, offset, includeSources);
}
return {
nodes: [],
Expand Down Expand Up @@ -875,7 +923,7 @@ export function registerGraphFunction(
(v) => typeof v === "string" && v.toLowerCase().includes(lower),
),
);
return paginate(matchingNodes, allEdges, 0, limit, offset);
return paginate(matchingNodes, allEdges, 0, limit, offset, includeSources);
}

if (data.startNodeId) {
Expand Down Expand Up @@ -917,11 +965,11 @@ export function registerGraphFunction(
}
}

return paginate(resultNodes, resultEdges, maxDepth, limit, offset);
return paginate(resultNodes, resultEdges, maxDepth, limit, offset, includeSources);
}

// Unreachable — noWalk branch handles the rest.
return paginate([], [], 0, limit, offset);
return paginate([], [], 0, limit, offset, includeSources);
},
);

Expand Down
8 changes: 8 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,10 @@ export interface GraphNode {
updatedAt?: string;
aliases?: string[];
stale?: boolean;
/** Present on graph-query responses: true length of sourceObservationIds
* before projection. The array itself is a recent sample unless the
* caller passed includeSources (upstream #1171). */
sourceObservationCount?: number;
}

export type GraphEdgeType =
Expand Down Expand Up @@ -445,6 +449,10 @@ export interface GraphEdge {
supersededBy?: string;
isLatest?: boolean;
stale?: boolean;
/** Present on graph-query responses: true length of sourceObservationIds
* before projection. The array itself is a recent sample unless the
* caller passed includeSources (upstream #1171). */
sourceObservationCount?: number;
}

export interface EdgeContext {
Expand Down
Loading