diff --git a/scripts/release-gate/graph-index.mjs b/scripts/release-gate/graph-index.mjs index 45c78099e3..06d22a3ee3 100644 --- a/scripts/release-gate/graph-index.mjs +++ b/scripts/release-gate/graph-index.mjs @@ -295,7 +295,7 @@ function persisted(p, scope) { } function graphFiles(p) { const dir = join(p.data, "state_store.db"); - return readdirSync(dir).filter((file) => decodeURIComponent(file).startsWith("mem:graph:")).map((file) => ({ scope: decodeURIComponent(file).replace(/\.bin$/, ""), bytes: statSync(join(dir, file)).size })); + return readdirSync(dir).filter((file) => file.endsWith(".bin") && decodeURIComponent(file).startsWith("mem:graph:")).map((file) => ({ scope: decodeURIComponent(file).replace(/\.bin$/, ""), bytes: statSync(join(dir, file)).size })); } function ordinaryReads(p, phase, requestCount = 10) { const summary = p.traceSummary.get(phase); diff --git a/src/functions/flow-compress.ts b/src/functions/flow-compress.ts index 3be1ca6e97..422fed8fbb 100644 --- a/src/functions/flow-compress.ts +++ b/src/functions/flow-compress.ts @@ -2,7 +2,10 @@ import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; import { KV, generateId } from "../state/schema.js"; import type { Action, ActionEdge, RoutineRun, MemoryProvider } from "../types.js"; -import { recordAudit } from "./audit.js"; +import { getAgentId } from "../config.js"; +import { logger } from "../logger.js"; +import { safeAudit } from "./audit.js"; +import { indexRecords } from "./search.js"; const FLOW_COMPRESS_SYSTEM = `You are a workflow summarizer. Given a completed action chain, produce a concise summary capturing: 1. The overall goal and outcome @@ -82,20 +85,40 @@ export function registerFlowCompressFunction( ); const summary = parseFlowSummary(response); const ts = new Date().toISOString(); + const agentId = getAgentId(); + const rawText = response + .replace(/<\/?(?:summary|goal|outcome|steps|discoveries|lesson)(?:\s[^>]*)?>/gi, " ") + .replace(/\s+/g, " ") + .trim(); + const project = + typeof data.project === "string" && data.project.length > 0 + ? doneActions.every((a) => a.project === data.project) + ? data.project + : undefined + : new Set(doneActions.map((a) => a.project)).size === 1 + ? doneActions[0].project + : undefined; const memory = { id: generateId("mem"), createdAt: ts, updatedAt: ts, type: "workflow" as const, - title: summary.goal || `Workflow: ${doneActions.length} actions`, - content: formatSummary(summary), + title: summary.goal || rawText.slice(0, 80) || `Workflow: ${doneActions.length} actions`, + // Fall back to the tag-stripped response so an unparseable or empty + // summary never produces an unsearchable memory or indexed markup. + content: + formatSummary(summary) || + rawText || + `Workflow with ${doneActions.length} completed actions`, concepts: extractConcepts(doneActions), files: extractFiles(doneActions), sessionIds: [], strength: 1.0, version: 1, isLatest: true, + ...(agentId ? { agentId } : {}), + ...(project !== undefined ? { project } : {}), metadata: { flowCompressed: true, actionCount: doneActions.length, @@ -104,12 +127,21 @@ export function registerFlowCompressFunction( }; await kv.set(KV.memories, memory.id, memory); - await recordAudit(kv, "compress", "mem::flow-compress", [memory.id], { + await safeAudit(kv, "compress", "mem::flow-compress", [memory.id], { action: "compress_flow", flowCompressed: true, actionCount: doneActions.length, - project: data.project, + project, }); + // Indexing failures are logged, not fatal: the KV write already + // committed and the restart rebuild is the backstop. + try { + await indexRecords([], [memory]); + } catch (err) { + logger.warn("Flow-compress indexing failed; restart rebuild will recover", { + error: err instanceof Error ? err.message : String(err), + }); + } return { success: true, @@ -166,7 +198,7 @@ function parseFlowSummary(response: string): { } { const extract = (tag: string): string => { const match = response.match( - new RegExp(`<${tag}>([\\s\\S]*?)`), + new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)`), ); return match ? match[1].trim() : ""; }; diff --git a/src/functions/graph-retrieval.ts b/src/functions/graph-retrieval.ts index 05f0704559..ee51f036d9 100644 --- a/src/functions/graph-retrieval.ts +++ b/src/functions/graph-retrieval.ts @@ -32,6 +32,7 @@ const MAX_EVIDENCE = 256; const MAX_OPERATIONS = 32; const MAX_RESULTS = 32; const FINAL_EVIDENCE_READS = 64; +const MAX_CANDIDATE_BATCH = 4; function compareText(a: string, b: string): number { return a === b ? 0 : a < b ? -1 : 1; } function nodeNames(node: GraphNode): string[] { @@ -215,15 +216,21 @@ export class GraphQuery { const characters = [...new Set([...name].filter((value) => value.trim()))]; if (characters.length > 16) this.truncated(); for (const [position, character] of characters.slice(0, 16).entries()) { + if (selected.size + examined.size >= reader.status.nodes) break; if (reader.remainingReads <= FINAL_EVIDENCE_READS || selected.size >= MAX_START_NODES) { this.truncated(); break; } const page = await reader.posting("character", character, { limit: position === 0 ? MAX_CANDIDATES : 16 }); if (page.cursor) this.truncated(); - for (const id of page.ids) { - if (selected.has(id) || examined.has(id)) continue; - if (examined.size >= MAX_CANDIDATES || reader.remainingReads <= FINAL_EVIDENCE_READS) { this.truncated(); break; } - examined.add(id); - const node = await reader.getNode(id); - if (node && !node.stale && nodeNames(node).some((value) => value.includes(name) || name.includes(value))) partialCandidates.set(id, node); + const candidates = page.ids.filter((id) => !selected.has(id) && !examined.has(id)); + for (let offset = 0; offset < candidates.length && !reader.exhausted && !this.controller.signal.aborted;) { + const room = Math.min(MAX_CANDIDATES - examined.size, reader.remainingReads - FINAL_EVIDENCE_READS); + if (room <= 0) { this.truncated(); break; } + const batch = candidates.slice(offset, offset + Math.min(MAX_CANDIDATE_BATCH, room)); + offset += batch.length; + for (const id of batch) examined.add(id); + const loaded = await Promise.all(batch.map(async (id) => ({ id, node: await reader.getNode(id) }))); + for (const { id, node } of loaded) { + if (node && !node.stale && nodeNames(node).some((value) => value.includes(name) || name.includes(value))) partialCandidates.set(id, node); + } } } } diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 307022d44c..62e7c7b304 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -3826,7 +3826,12 @@ export function registerApiTriggers( const authErr = checkAuth(req, secret); if (authErr) return authErr; try { - const result = await sdk.trigger({ function_id: "mem::flow-compress", payload: req.body || {} }); + const body = req.body ?? {}; + const result = await sdk.trigger({ function_id: "mem::flow-compress", payload: { + runId: body.runId, + actionIds: body.actionIds, + project: body.project, + } }); return { status_code: 200, body: result }; } catch { return { diff --git a/test/flow-compress-index.test.ts b/test/flow-compress-index.test.ts new file mode 100644 index 0000000000..6636deba28 --- /dev/null +++ b/test/flow-compress-index.test.ts @@ -0,0 +1,191 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { registerFlowCompressFunction } from "../src/functions/flow-compress.js"; +import { getSearchIndex } from "../src/functions/search.js"; +import type { Action, Memory, MemoryProvider } from "../src/types.js"; +import type { IIIClient } from "iii-sdk"; +import type { StateKV } from "../src/state/kv.js"; + +vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); + +function doneAction(id: string, project = "proj-a"): Action { + const ts = "2026-11-20T00:00:00.000Z"; + return { + id, + title: `Finish ${id}`, + description: `Work for ${id}`, + status: "done", + priority: 1, + createdAt: ts, + updatedAt: ts, + createdBy: "agent-a", + project, + tags: [], + sourceObservationIds: [], + sourceMemoryIds: [], + result: "shipped", + }; +} + +const SUMMARY_XML = [ + "", + "Quasar deploy pipeline", + "Pipeline shipped", + "Built image, pushed, rolled out", + "Registry needed auth", + "Cache credentials early", + "", +].join("\n"); + +function fixture( + summarize: (system: string, prompt: string) => Promise = async () => SUMMARY_XML, + extraScopes: Record> = {}, +) { + const store = new Map>([ + ["mem:actions", new Map([["act_1", doneAction("act_1")]])], + ...Object.entries(extraScopes), + ]); + const kv = { + get: async (scope: string, id: string) => store.get(scope)?.get(id) ?? null, + set: async (scope: string, id: string, value: unknown) => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(id, value); + return value; + }, + list: async (scope: string) => [...(store.get(scope)?.values() ?? [])], + }; + let handler: (args: object) => Promise<{ success: boolean; memoryId?: string; compressed?: number }>; + const sdk = { registerFunction: (_: string, fn: typeof handler) => { handler = fn; } }; + const provider = { summarize: vi.fn(summarize) }; + registerFlowCompressFunction(sdk as unknown as IIIClient, kv as unknown as StateKV, provider as unknown as MemoryProvider); + const memory = (id: string) => store.get("mem:memories")?.get(id) as Memory | undefined; + return { call: (args: object) => handler(args), provider, memory }; +} + +describe("flow-compressed memories", () => { + afterEach(() => getSearchIndex().clear()); + + it("are searchable immediately after mem::flow-compress returns", async () => { + const f = fixture(); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(true); + expect(result.memoryId).toMatch(/^mem_/); + expect(f.provider.summarize).toHaveBeenCalledOnce(); + + const idx = getSearchIndex(); + expect(idx.has(result.memoryId!)).toBe(true); + expect(idx.layerOf(result.memoryId!)).toBe("memory"); + expect( + idx.search("quasar deploy pipeline", 5, "memory").map((r) => r.obsId), + ).toContain(result.memoryId); + expect(f.memory(result.memoryId!)).toMatchObject({ + id: result.memoryId, + title: "Quasar deploy pipeline", + isLatest: true, + metadata: expect.objectContaining({ flowCompressed: true }), + }); + }); + + it("stamps the agent so scoped recall can find it", async () => { + vi.stubEnv("AGENT_ID", "agent-x"); + try { + const f = fixture(); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(true); + expect(f.memory(result.memoryId!)!.agentId).toBe("agent-x"); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("stay searchable when the provider output has no summary tags", async () => { + const f = fixture( + async () => "quasar deploy failed where left < threshold && threshold > right on retry", + ); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(true); + expect(result.memoryId).toBeTruthy(); + + expect(getSearchIndex().has(result.memoryId!)).toBe(true); + expect(f.memory(result.memoryId!)!.content).toContain("quasar"); + expect(f.memory(result.memoryId!)!.content).toContain("left < threshold && threshold > right"); + }); + + it("keeps long fallback summaries searchable past the embedding input limit", async () => { + const response = "deployment notes ".repeat(1100) + "Quasar rollback checkpoint"; + const f = fixture(async () => response); + const result = await f.call({ actionIds: ["act_1"] }); + + expect(result.success).toBe(true); + const stored = f.memory(result.memoryId!)!; + expect(stored.title.length).toBeLessThanOrEqual(80); + expect(stored.content).toBe(response); + expect( + getSearchIndex().search("quasar rollback checkpoint", 5, "memory").map((r) => r.obsId), + ).toContain(result.memoryId); + }); + + it("parses attributed summary tags instead of dropping them", async () => { + const f = fixture( + async () => + 'Quasar pipeline' + + 'built and pushed', + ); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(true); + + const stored = f.memory(result.memoryId!)!; + expect(stored.title).toBe("Quasar pipeline"); + expect(stored.content).toContain("built and pushed"); + expect(stored.content).not.toContain("<"); + expect(getSearchIndex().has(result.memoryId!)).toBe(true); + }); + + it("returns failure and indexes nothing when summarization fails", async () => { + const f = fixture(async () => { throw new Error("provider down"); }); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(false); + expect(result.memoryId).toBeUndefined(); + expect(getSearchIndex().size).toBe(0); + }); + + it("falls back to generic content when the provider returns nothing", async () => { + const f = fixture(async () => ""); + const result = await f.call({ actionIds: ["act_1"] }); + expect(result.success).toBe(true); + expect(result.memoryId).toBeTruthy(); + expect(getSearchIndex().has(result.memoryId!)).toBe(true); + expect(f.memory(result.memoryId!)!.content).toContain("completed actions"); + }); + + it("stamps the project only when project selected the actions", async () => { + const byProject = fixture(); + const scoped = await byProject.call({ project: "proj-a" }); + expect(scoped.success).toBe(true); + expect(byProject.memory(scoped.memoryId!)!.project).toBe("proj-a"); + + const byIds = fixture(); + const unscoped = await byIds.call({ actionIds: ["act_1"], project: "other" }); + expect(unscoped.success).toBe(true); + expect(byIds.memory(unscoped.memoryId!)!.project).toBeUndefined(); + expect(getSearchIndex().has(unscoped.memoryId!)).toBe(true); + + const matching = fixture(); + const confirmed = await matching.call({ actionIds: ["act_1"], project: "proj-a" }); + expect(confirmed.success).toBe(true); + expect(matching.memory(confirmed.memoryId!)!.project).toBe("proj-a"); + + const run = { + id: "run_1", routineId: "r", status: "completed", startedAt: "2026-11-20T00:00:00.000Z", + actionIds: ["act_1"], stepStatus: {}, initiatedBy: "agent-a", + }; + const byRun = fixture(undefined, { "mem:routine-runs": new Map([["run_1", run]]) }); + const fromRun = await byRun.call({ runId: "run_1", project: "other" }); + expect(fromRun.success).toBe(true); + expect(byRun.memory(fromRun.memoryId!)!.project).toBeUndefined(); + + const inferred = fixture(undefined, { "mem:routine-runs": new Map([["run_1", run]]) }); + const fromRunNoScope = await inferred.call({ runId: "run_1" }); + expect(fromRunNoScope.success).toBe(true); + expect(inferred.memory(fromRunNoScope.memoryId!)!.project).toBe("proj-a"); + }); +}); diff --git a/test/graph-index-gate-files.test.ts b/test/graph-index-gate-files.test.ts new file mode 100644 index 0000000000..91482c758c --- /dev/null +++ b/test/graph-index-gate-files.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { runInNewContext } from "node:vm"; + +const script = readFileSync(new URL("../scripts/release-gate/graph-index.mjs", import.meta.url), "utf8"); +const start = script.indexOf("function graphFiles(p) {"); +const end = script.indexOf("\nfunction ordinaryReads(", start); + +function graphFiles(files: string[], stat: (path: string) => { size: number }) { + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + return runInNewContext(`(${script.slice(start, end)})`, { + join, + readdirSync: () => files, + statSync: stat, + })({ data: "/fixture" }); +} + +describe("packaged graph gate file diagnostics", () => { + it("ignores a temporary graph file removed by an atomic rename", () => { + const stat = vi.fn((path: string) => { + if (path.endsWith(".tmp")) throw Object.assign(new Error("file renamed"), { code: "ENOENT" }); + return { size: 128 }; + }); + expect(graphFiles([ + "mem%3Agraph%3Aretrieval-data.bin", + "mem%3Agraph%3Aretrieval-data.bin.tmp", + "mem%3Amemories.bin", + ], stat)).toEqual([{ scope: "mem:graph:retrieval-data", bytes: 128 }]); + expect(stat).toHaveBeenCalledOnce(); + }); + + it("still reports stat failures for committed graph files", () => { + const error = Object.assign(new Error("permission denied"), { code: "EACCES" }); + expect(() => graphFiles(["mem%3Agraph%3Aretrieval-data.bin"], () => { throw error; })).toThrow(error); + }); +}); diff --git a/test/graph-retrieval-latency.test.ts b/test/graph-retrieval-latency.test.ts new file mode 100644 index 0000000000..3e6ba53668 --- /dev/null +++ b/test/graph-retrieval-latency.test.ts @@ -0,0 +1,60 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HybridSearch } from "../src/state/hybrid-search.js"; +import { SearchIndex } from "../src/state/search-index.js"; +import { memoryToObservation } from "../src/state/memory-utils.js"; +import { KV } from "../src/state/schema.js"; +import { initializeGraphIndex, reconcileGraphIndexRecord } from "../src/state/graph-retrieval-index.js"; +import { createGraphIndexState } from "./helpers/graph-index-state.js"; +import type { Memory } from "../src/types.js"; + +afterEach(() => vi.useRealTimers()); + +describe("graph-only recall with storage latency", () => { + it("retains exact memory evidence while examining unrelated partial-name candidates", async () => { + const { kv, intercept } = createGraphIndexState(); + const timestamp = "2026-10-14T00:00:00.000Z"; + const memory: Memory = { + id: "mem_durable", createdAt: timestamp, updatedAt: timestamp, + type: "fact", title: "Indigo durable capture", content: "Stored evidence", + concepts: [], files: [], sessionIds: [], strength: 5, version: 1, + isLatest: true, + }; + const index = new SearchIndex(); + index.add(memoryToObservation(memory), "memory"); + await kv.set(KV.memories, memory.id, memory); + await initializeGraphIndex(kv, { status: "ready" }); + for (let i = 0; i < 100; i++) { + const id = `node-${i}`; + await reconcileGraphIndexRecord(kv, "node", id, { + id, name: i === 0 ? "OnyxRestored" : `Cobalt${i}`, + type: "concept", properties: {}, createdAt: timestamp, + sourceObservationIds: i === 0 ? [memory.id] : ["legacy_observation"], + }); + } + + vi.useFakeTimers(); + let active = 0; + let maximumActive = 0; + intercept(async (id) => { + if (id !== "state::get") return; + active++; + maximumActive = Math.max(maximumActive, active); + try { + await new Promise((resolve) => setTimeout(resolve, 3)); + } finally { + active--; + } + }); + const hybrid = new HybridSearch(index, null, null, kv); + expect(index.search("OnyxRestored", 5, "memory")).toEqual([]); + const search = hybrid.search("OnyxRestored", 5, "memory"); + await vi.runAllTimersAsync(); + const results = await search; + + expect(results.map((result) => result.observation.id)).toContain(memory.id); + expect(results.find((result) => result.observation.id === memory.id)?.graphScore).toBeGreaterThan(0); + expect(maximumActive).toBeLessThanOrEqual(4); + expect(active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/test/graph-retrieval.test.ts b/test/graph-retrieval.test.ts index f6a00f9826..7fd7aa283a 100644 --- a/test/graph-retrieval.test.ts +++ b/test/graph-retrieval.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi } from "vitest"; import { GraphRetrieval } from "../src/functions/graph-retrieval.js"; -import { getGraphIndexStatus, initializeGraphIndex, reconcileGraphIndexRecord } from "../src/state/graph-retrieval-index.js"; +import { getGraphIndexStatus, initializeGraphIndex, reconcileGraphIndexRecord, GraphIndexReader } from "../src/state/graph-retrieval-index.js"; import { indexObservationSession } from "../src/state/obs-index.js"; import { KV } from "../src/state/schema.js"; import type { GraphNode, GraphEdge } from "../src/types.js"; @@ -571,3 +571,64 @@ it("does not replay cached operations after the caller cancels its query", async query.cancel(); await expect(query.searchByEntities(["Cobalt"])).rejects.toThrow(); }); + +it("keeps deterministic partial candidate order when node reads resolve out of order", async () => { + const nodes = [ + makeNode("p-zeta", "zeta-auth", "concept", ["obs_zeta"]), + makeNode("p-alpha", "alpha-auth", "concept", ["obs_alpha"]), + ...Array.from({ length: 6 }, (_, index) => makeNode(`fill-${index}`, `hat-${index}`, "concept", [`fill_${index}`])), + ]; + const kv = await mockKV(nodes, []); + const pending: Array<() => void> = []; + const get = kv.get; + vi.spyOn(kv, "get").mockImplementation(async (scope, key) => { + const value = await get(scope, key); + if (scope === KV.graphRetrievalData && key.includes(":record:node:")) await new Promise((resolve) => pending.push(resolve)); + return value; + }); + const search = new GraphRetrieval(kv as never).searchByEntities(["auth"], 0); + let settled = false; + void search.then(() => { settled = true; }, () => { settled = true; }); + while (!settled) { + for (const resolve of pending.splice(0).reverse()) resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); + } + expect((await search).map((result) => result.obsId)).toEqual(["obs_alpha", "obs_zeta"]); +}); + +it("does not dispatch further partial candidate batches after the caller cancels its query", async () => { + const nodes = Array.from({ length: 8 }, (_, index) => makeNode(`fill-${index}`, `atlas-${index}`, "concept", [`obs_${index}`])); + const kv = await mockKV(nodes, []); + const query = new GraphRetrieval(kv as never).createQuery(); + const getNode = vi.spyOn(GraphIndexReader.prototype, "getNode"); + let canceled = false; + const get = kv.get; + vi.spyOn(kv, "get").mockImplementation(async (scope, key) => { + if (!canceled && scope === KV.graphRetrievalData && key.includes(":record:node:")) { canceled = true; query.cancel(); } + return get(scope, key); + }); + try { + await expect(query.searchByEntities(["auth"], 0)).rejects.toThrow(); + expect(canceled).toBe(true); + expect(getNode.mock.calls.length).toBeLessThanOrEqual(4); + } finally { + getNode.mockRestore(); + } +}); + +it("stops requesting character postings once every indexed node is covered", async () => { + const nodes = [ + makeNode("exact", "auth", "function", ["obs_auth"]), + makeNode("partial", "auth-handler", "function", ["obs_handler"]), + makeNode("other", "atlas", "concept", ["obs_atlas"]), + ]; + const kv = await mockKV(nodes, []); + const posting = vi.spyOn(GraphIndexReader.prototype, "posting"); + try { + const results = await new GraphRetrieval(kv as never).searchByEntities(["auth"], 0); + expect(results.map((result) => result.obsId)).toEqual(["obs_auth", "obs_handler"]); + expect(posting.mock.calls.filter((call) => call[0] === "character").map((call) => call[1])).toEqual(["a"]); + } finally { + posting.mockRestore(); + } +});