From d5741aed8bb837d4e1c86897b9be12fbfd786316 Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Wed, 15 Jul 2026 11:41:02 +0800 Subject: [PATCH 1/6] fix(summarize): dedupe double-triggered session summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single stop-hook invocation used to run mem::summarize twice: once via the direct POST /agentmemory/summarize, and once via /agentmemory/session/end -> event::session::stopped. Each run re-summarized the entire session from scratch, so every session stop cost 2x full-history LLM calls (observed: duplicate ~230KB requests per stop, growing as the session accumulated observations). - stop hook: drop the direct summarize call; session/end already fans out to the summarize path. - mem::summarize: serialize per-session runs with withKeyedLock (same pattern as mem::observe) and short-circuit when the stored summary's observationCount already matches — observations are append-only, so an unchanged count means the LLM input would be identical. --- plugin/scripts/stop.mjs | 10 +- src/functions/summarize.ts | 280 ++++++++++++++++++++----------------- src/hooks/stop.ts | 10 +- test/summarize.test.ts | 61 ++++++++ 4 files changed, 217 insertions(+), 144 deletions(-) diff --git a/plugin/scripts/stop.mjs b/plugin/scripts/stop.mjs index 03d30c64f3..ecbd98379f 100755 --- a/plugin/scripts/stop.mjs +++ b/plugin/scripts/stop.mjs @@ -23,12 +23,6 @@ async function main() { } if (isSdkChildContext(data)) return; const sessionId = data.session_id || data.sessionId || "unknown"; - fetch(`${REST_URL}/agentmemory/summarize`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ sessionId }), - signal: AbortSignal.timeout(12e4) - }).catch(() => {}); fetch(`${REST_URL}/agentmemory/session/end`, { method: "POST", headers: authHeaders(), @@ -38,7 +32,7 @@ async function main() { setTimeout(() => process.exit(0), 1500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=stop.mjs.map \ No newline at end of file diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 4c501ca8cf..12a3dbfc5b 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -7,6 +7,7 @@ import type { } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; import { SUMMARY_SYSTEM, buildSummaryPrompt, @@ -240,159 +241,180 @@ export function registerSummarizeFunction( } const sessionId = data.sessionId.trim(); - const session = await kv.get(KV.sessions, sessionId); - if (!session) { - logger.warn("Session not found for summarize", { - sessionId, - }); - return { success: false, error: "session_not_found" }; - } + // Serialize per-session runs: the stop path and the explicit API can + // race, and a second concurrent run would redo the same LLM work. + // See the up-to-date short-circuit below for the sequential case. + return withKeyedLock(`summarize:${sessionId}`, async () => { + const session = await kv.get(KV.sessions, sessionId); + if (!session) { + logger.warn("Session not found for summarize", { + sessionId, + }); + return { success: false, error: "session_not_found" }; + } - const observations = await kv.list( - KV.observations(sessionId), - ); - const compressed = observations.filter((o) => o.title); + const observations = await kv.list( + KV.observations(sessionId), + ); + const compressed = observations.filter((o) => o.title); - if (compressed.length === 0) { - logger.info("No observations to summarize", { - sessionId, - }); - return { success: false, error: "no_observations" }; - } + if (compressed.length === 0) { + logger.info("No observations to summarize", { + sessionId, + }); + return { success: false, error: "no_observations" }; + } - if (provider.name === "noop") { - logger.info("Summarize skipped — no LLM provider configured", { + // Observations are append-only, so an unchanged count means the + // LLM input would be identical to the previous run. Skipping avoids + // re-summarizing long-lived sessions (e.g. heartbeat loops) from + // scratch on every stop when nothing new happened. + const existing = await kv.get( + KV.summaries, sessionId, - }); - return { - success: false, - error: "no_provider", - reason: - "No LLM provider key set; Summarize is a no-op. Set ANTHROPIC_API_KEY (or GEMINI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env to enable.", - }; - } + ); + if (existing && existing.observationCount === compressed.length) { + logger.info("Summary already up to date — skipping", { + sessionId, + observationCount: compressed.length, + }); + return { success: true, summary: existing, skipped: true }; + } - try { - // #783: chunk-level produceSummaryXml retries internally, but - // the final merge used to parse once and bail. Wrap the - // produce-and-parse pair in the same 2-attempt loop so a - // markdown-wrapped or otherwise wrapped response gets a - // second roll-of-the-dice instead of dropping the summary. - let summary: SessionSummary | null = null; - let response = ""; - let mode = "single"; - let chunks = 1; - for (let attempt = 1; attempt <= 2; attempt++) { - const produced = await produceSummaryXml( - provider, - compressed, + if (provider.name === "noop") { + logger.info("Summarize skipped — no LLM provider configured", { sessionId, - session.project, - ); - response = produced.response; - mode = produced.mode; - chunks = produced.chunks; - if (!response || !response.trim()) { - logger.warn("Empty provider response on summarize", { + }); + return { + success: false, + error: "no_provider", + reason: + "No LLM provider key set; Summarize is a no-op. Set ANTHROPIC_API_KEY (or GEMINI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env to enable.", + }; + } + + try { + // #783: chunk-level produceSummaryXml retries internally, but + // the final merge used to parse once and bail. Wrap the + // produce-and-parse pair in the same 2-attempt loop so a + // markdown-wrapped or otherwise wrapped response gets a + // second roll-of-the-dice instead of dropping the summary. + let summary: SessionSummary | null = null; + let response = ""; + let mode = "single"; + let chunks = 1; + for (let attempt = 1; attempt <= 2; attempt++) { + const produced = await produceSummaryXml( + provider, + compressed, sessionId, - provider: provider.name, - mode, - chunks, - observationCount: compressed.length, - attempt, - }); - continue; + session.project, + ); + response = produced.response; + mode = produced.mode; + chunks = produced.chunks; + if (!response || !response.trim()) { + logger.warn("Empty provider response on summarize", { + sessionId, + provider: provider.name, + mode, + chunks, + observationCount: compressed.length, + attempt, + }); + continue; + } + summary = parseSummaryXml( + response, + sessionId, + session.project, + compressed.length, + ); + if (summary) break; + logger.warn("Failed to parse summary XML", { sessionId, attempt }); } - summary = parseSummaryXml( - response, - sessionId, - session.project, - compressed.length, + + if (!response || !response.trim()) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + return { success: false, error: "empty_provider_response" }; + } + + if (!summary) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + return { success: false, error: "parse_failed" }; + } + + const summaryForValidation = { + title: summary.title, + narrative: summary.narrative, + keyDecisions: summary.keyDecisions, + filesModified: summary.filesModified, + concepts: summary.concepts, + }; + const validation = validateOutput( + SummaryOutputSchema, + summaryForValidation, + "mem::summarize", ); - if (summary) break; - logger.warn("Failed to parse summary XML", { sessionId, attempt }); - } - if (!response || !response.trim()) { - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); + if (!validation.valid) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + logger.warn("Summary validation failed", { + sessionId, + errors: validation.result.errors, + }); + return { success: false, error: "validation_failed" }; } - return { success: false, error: "empty_provider_response" }; - } - if (!summary) { + const qualityScore = scoreSummary(summaryForValidation); + + await kv.set(KV.summaries, sessionId, summary); + await safeAudit(kv, "compress", "mem::summarize", [sessionId], { + title: summary.title, + observationCount: compressed.length, + }); + const latencyMs = Date.now() - startMs; if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); + await metricsStore.record( + "mem::summarize", + latencyMs, + true, + qualityScore, + ); } - return { success: false, error: "parse_failed" }; - } - const summaryForValidation = { - title: summary.title, - narrative: summary.narrative, - keyDecisions: summary.keyDecisions, - filesModified: summary.filesModified, - concepts: summary.concepts, - }; - const validation = validateOutput( - SummaryOutputSchema, - summaryForValidation, - "mem::summarize", - ); + logger.info("Session summarized", { + sessionId, + title: summary.title, + decisions: summary.keyDecisions.length, + qualityScore, + valid: validation.valid, + }); - if (!validation.valid) { + return { success: true, summary, qualityScore }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); const latencyMs = Date.now() - startMs; if (metricsStore) { await metricsStore.record("mem::summarize", latencyMs, false); } - logger.warn("Summary validation failed", { + logger.error("Summarize failed", { sessionId, - errors: validation.result.errors, + error: msg, }); - return { success: false, error: "validation_failed" }; - } - - const qualityScore = scoreSummary(summaryForValidation); - - await kv.set(KV.summaries, sessionId, summary); - await safeAudit(kv, "compress", "mem::summarize", [sessionId], { - title: summary.title, - observationCount: compressed.length, - }); - - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record( - "mem::summarize", - latencyMs, - true, - qualityScore, - ); + return { success: false, error: msg }; } - - logger.info("Session summarized", { - sessionId, - title: summary.title, - decisions: summary.keyDecisions.length, - qualityScore, - valid: validation.valid, - }); - - return { success: true, summary, qualityScore }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); - } - logger.error("Summarize failed", { - sessionId, - error: msg, - }); - return { success: false, error: msg }; - } + }); }, ); } diff --git a/src/hooks/stop.ts b/src/hooks/stop.ts index 4103796e0f..763947adac 100644 --- a/src/hooks/stop.ts +++ b/src/hooks/stop.ts @@ -39,13 +39,9 @@ async function main() { const sessionId = ((data.session_id || data.sessionId) as string) || "unknown"; - fetch(`${REST_URL}/agentmemory/summarize`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ sessionId }), - signal: AbortSignal.timeout(120000), - }).catch(() => {}); - + // Summarize is NOT called directly here: /agentmemory/session/end fans out + // event::session::stopped, whose handler already runs mem::summarize. + // Calling both used to double every summarize LLM run. fetch(`${REST_URL}/agentmemory/session/end`, { method: "POST", headers: authHeaders(), diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 4723da8781..818916562c 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -480,3 +480,64 @@ describe("mem::summarize chunking", () => { expect(result.error).toBe("parse_failed"); }); }); + +describe("mem::summarize dedup (double-trigger guard)", () => { + it("skips the LLM when observation count is unchanged since last summary", async () => { + const provider = makeProvider([summaryXml({ title: "First run" })]); + const { handler } = await setupHandler({ + sessionId: "ses_dedup", + obsCount: 10, + provider, + }); + + const first: any = await handler({ sessionId: "ses_dedup" }); + expect(first.success).toBe(true); + expect(provider.calls).toHaveLength(1); + + const second: any = await handler({ sessionId: "ses_dedup" }); + expect(second.success).toBe(true); + expect(second.skipped).toBe(true); + expect(second.summary.title).toBe("First run"); + expect(provider.calls).toHaveLength(1); // no second LLM run + }); + + it("collapses concurrent duplicate calls into a single LLM run", async () => { + const provider = makeProvider([summaryXml({ title: "Only run" })]); + const { handler } = await setupHandler({ + sessionId: "ses_race", + obsCount: 10, + provider, + }); + + const [a, b]: any[] = await Promise.all([ + handler({ sessionId: "ses_race" }), + handler({ sessionId: "ses_race" }), + ]); + + expect(a.success).toBe(true); + expect(b.success).toBe(true); + expect(provider.calls).toHaveLength(1); + }); + + it("re-summarizes when new observations arrived since last summary", async () => { + const provider = makeProvider([ + summaryXml({ title: "First run" }), + summaryXml({ title: "Second run" }), + ]); + const { handler, kv } = await setupHandler({ + sessionId: "ses_grow", + obsCount: 10, + provider, + }); + + await handler({ sessionId: "ses_grow" }); + const extra = makeObs(10, "ses_grow"); + await kv.set("obs:ses_grow", extra.id, extra); + + const result: any = await handler({ sessionId: "ses_grow" }); + expect(result.success).toBe(true); + expect(result.skipped).toBeUndefined(); + expect(result.summary.title).toBe("Second run"); + expect(provider.calls).toHaveLength(2); + }); +}); From 18fe9a19839df4a722500c8475acc0c01399211f Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Wed, 15 Jul 2026 14:11:07 +0800 Subject: [PATCH 2/6] =?UTF-8?q?fix(summarize):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20fingerprint=20skip,=20lock=20timeout,=20hook=20exit?= =?UTF-8?q?=20timer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace the observationCount-based skip with an input fingerprint (sha256 over ordered observation ids + timestamps). Count alone is not safe: evict/auto-forget can delete observations, so the set can change while the count returns to a previous value. Old summaries have no fingerprint and simply re-summarize once. - Bound how long one run may hold the per-session lock (SUMMARIZE_LOCK_TIMEOUT_MS, default 5 min): anthropic/agent-sdk providers have no request timeout, and a hung call must not block every later summarize for that session. - Align the stop hook forced-exit timer with other single-request hooks (1500ms -> 500ms) now that it only sends one request. --- plugin/scripts/stop.mjs | 2 +- src/functions/input-fingerprint.ts | 14 +++++++ src/functions/summarize.ts | 63 ++++++++++++++++++++++++++---- src/hooks/stop.ts | 2 +- src/types.ts | 4 ++ test/summarize.test.ts | 53 +++++++++++++++++++++++++ 6 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 src/functions/input-fingerprint.ts diff --git a/plugin/scripts/stop.mjs b/plugin/scripts/stop.mjs index ecbd98379f..7c4b21fc64 100755 --- a/plugin/scripts/stop.mjs +++ b/plugin/scripts/stop.mjs @@ -29,7 +29,7 @@ async function main() { body: JSON.stringify({ sessionId }), signal: AbortSignal.timeout(5e3) }).catch(() => {}); - setTimeout(() => process.exit(0), 1500).unref(); + setTimeout(() => process.exit(0), 500).unref(); } main(); //#endregion diff --git a/src/functions/input-fingerprint.ts b/src/functions/input-fingerprint.ts new file mode 100644 index 0000000000..b87c608f8a --- /dev/null +++ b/src/functions/input-fingerprint.ts @@ -0,0 +1,14 @@ +import { createHash } from "node:crypto"; +import type { CompressedObservation } from "../types.js"; + +// Stable fingerprint of an exact observation set, used to skip LLM re-runs +// (summarize, graph extraction) when the input is unchanged. Count alone is +// not enough: evict/auto-forget can delete observations, so the set can +// change while the count returns to a previous value. +export function computeInputFingerprint( + observations: CompressedObservation[], +): string { + const h = createHash("sha256"); + for (const o of observations) h.update(`${o.id} ${o.timestamp} `); + return h.digest("hex").slice(0, 32); +} diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 12a3dbfc5b..fc2dbc86b4 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -15,6 +15,7 @@ import { buildReducePrompt, } from "../prompts/summary.js"; import { getXmlTag, getXmlChildren } from "../prompts/xml.js"; +import { computeInputFingerprint } from "./input-fingerprint.js"; import { SummaryOutputSchema } from "../eval/schemas.js"; import { validateOutput } from "../eval/validator.js"; import { scoreSummary } from "../eval/quality.js"; @@ -51,6 +52,44 @@ function getChunkConcurrency(): number { return Number.isFinite(n) && n > 0 ? n : CHUNK_CONCURRENCY_DEFAULT; } +// Upper bound on how long one summarize run may hold the per-session lock. +// Some providers (anthropic, agent-sdk) have no request timeout of their +// own; without a bound a hung LLM call would block every later summarize +// for that session forever. Generous enough for chunked map-reduce runs. +const LOCK_TIMEOUT_MS_DEFAULT = 300_000; + +function getLockTimeoutMs(): number { + const raw = process.env.SUMMARIZE_LOCK_TIMEOUT_MS; + if (!raw) return LOCK_TIMEOUT_MS_DEFAULT; + const n = parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : LOCK_TIMEOUT_MS_DEFAULT; +} + +// Resolves with a timeout marker instead of rejecting so the caller gets a +// normal { success: false } result. The underlying run keeps going in the +// background, but the keyed lock is released. +function raceLockTimeout( + work: Promise, + ms: number, + sessionId: string, +): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise<{ success: false; error: string }>((resolve) => { + timer = setTimeout(() => { + logger.warn("Summarize lock timeout — releasing session lock", { + sessionId, + timeoutMs: ms, + }); + resolve({ success: false, error: "summarize_timeout" }); + }, ms); + timer.unref?.(); + }); + return Promise.race([ + work.finally(() => clearTimeout(timer)), + timeout, + ]); +} + // One chunk call with retry-once. Returns null when both attempts fail — // whether by parse failure, provider 4xx (content rejected by upstream // filters), or transient network/5xx errors that didn't recover on retry. @@ -244,7 +283,7 @@ export function registerSummarizeFunction( // Serialize per-session runs: the stop path and the explicit API can // race, and a second concurrent run would redo the same LLM work. // See the up-to-date short-circuit below for the sequential case. - return withKeyedLock(`summarize:${sessionId}`, async () => { + const runSummarize = async () => { const session = await kv.get(KV.sessions, sessionId); if (!session) { logger.warn("Session not found for summarize", { @@ -265,15 +304,18 @@ export function registerSummarizeFunction( return { success: false, error: "no_observations" }; } - // Observations are append-only, so an unchanged count means the - // LLM input would be identical to the previous run. Skipping avoids - // re-summarizing long-lived sessions (e.g. heartbeat loops) from - // scratch on every stop when nothing new happened. + // An unchanged input fingerprint means the LLM input would be + // identical to the previous run. Skipping avoids re-summarizing + // long-lived sessions (e.g. heartbeat loops) from scratch on every + // stop when nothing new happened. Summaries written before this + // field existed have no fingerprint and never match, so they + // re-summarize once and pick one up. + const inputFingerprint = computeInputFingerprint(compressed); const existing = await kv.get( KV.summaries, sessionId, ); - if (existing && existing.observationCount === compressed.length) { + if (existing && existing.inputFingerprint === inputFingerprint) { logger.info("Summary already up to date — skipping", { sessionId, observationCount: compressed.length, @@ -377,7 +419,8 @@ export function registerSummarizeFunction( const qualityScore = scoreSummary(summaryForValidation); - await kv.set(KV.summaries, sessionId, summary); + summary.inputFingerprint = inputFingerprint; + await kv.set(KV.summaries, sessionId, summary); await safeAudit(kv, "compress", "mem::summarize", [sessionId], { title: summary.title, observationCount: compressed.length, @@ -414,7 +457,11 @@ export function registerSummarizeFunction( }); return { success: false, error: msg }; } - }); + }; + + return withKeyedLock(`summarize:${sessionId}`, () => + raceLockTimeout(runSummarize(), getLockTimeoutMs(), sessionId), + ); }, ); } diff --git a/src/hooks/stop.ts b/src/hooks/stop.ts index 763947adac..42c761466f 100644 --- a/src/hooks/stop.ts +++ b/src/hooks/stop.ts @@ -49,7 +49,7 @@ async function main() { signal: AbortSignal.timeout(5000), }).catch(() => {}); - setTimeout(() => process.exit(0), 1500).unref(); + setTimeout(() => process.exit(0), 500).unref(); } main(); diff --git a/src/types.ts b/src/types.ts index 6797dfaf99..5cf5d9d2d1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -114,6 +114,10 @@ export interface SessionSummary { filesModified: string[]; concepts: string[]; observationCount: number; + // Fingerprint of the exact observation set this summary was built from; + // used by mem::summarize to skip re-runs with identical input. Absent on + // summaries written by older versions. + inputFingerprint?: string; } export type HookType = diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 818916562c..64e16f5e14 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -519,6 +519,59 @@ describe("mem::summarize dedup (double-trigger guard)", () => { expect(provider.calls).toHaveLength(1); }); + it("re-summarizes when the observation set changed but the count did not", async () => { + const provider = makeProvider([ + summaryXml({ title: "First run" }), + summaryXml({ title: "Second run" }), + ]); + const { handler, kv } = await setupHandler({ + sessionId: "ses_swap", + obsCount: 10, + provider, + }); + + await handler({ sessionId: "ses_swap" }); + + // evict/auto-forget can delete observations; a new one can then bring + // the count back to its previous value with different content. + await kv.delete("obs:ses_swap", "obs_0"); + const extra = makeObs(10, "ses_swap"); + await kv.set("obs:ses_swap", extra.id, extra); + + const result: any = await handler({ sessionId: "ses_swap" }); + expect(result.success).toBe(true); + expect(result.skipped).toBeUndefined(); + expect(result.summary.title).toBe("Second run"); + expect(provider.calls).toHaveLength(2); + }); + + it("releases the session lock when a run exceeds SUMMARIZE_LOCK_TIMEOUT_MS", async () => { + process.env.SUMMARIZE_LOCK_TIMEOUT_MS = "50"; + try { + const provider: MemoryProvider = { + name: "hung", + compress: async () => "", + summarize: () => new Promise(() => {}), // never settles + }; + const { handler } = await setupHandler({ + sessionId: "ses_hung", + obsCount: 10, + provider, + }); + + const result: any = await handler({ sessionId: "ses_hung" }); + expect(result.success).toBe(false); + expect(result.error).toBe("summarize_timeout"); + + // The lock must be free again: a follow-up call proceeds instead of + // queueing behind the hung run. + const again: any = await handler({ sessionId: "ses_hung" }); + expect(again.error).toBe("summarize_timeout"); + } finally { + delete process.env.SUMMARIZE_LOCK_TIMEOUT_MS; + } + }); + it("re-summarizes when new observations arrived since last summary", async () => { const provider = makeProvider([ summaryXml({ title: "First run" }), From b6a002b9456eac0e9d53d273c20d5c8059ebb5e9 Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Wed, 15 Jul 2026 16:07:18 +0800 Subject: [PATCH 3/6] fix(events): skip graph re-extraction when the observation set is unchanged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The session-stopped handler sent the session's full compressed observation set to mem::graph-extract on every stop. Sessions that stop repeatedly without new activity (heartbeat-style loops) re-extracted the identical set each time — for a 500-observation session that is a ~50K-token LLM call per stop. Fingerprint the set (same helper as the summarize skip) and store it per session; trigger extraction only when it changed. --- src/state/schema.ts | 4 ++++ src/triggers/events.ts | 32 +++++++++++++++++++++---- test/session-end-triggers-graph.test.ts | 25 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/state/schema.ts b/src/state/schema.ts index cb29d41ad8..75c756605e 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -22,6 +22,10 @@ export const KV = { // Single fixed key ("current") so writes are read-modify-write under // the same keyed mutex as graph-extract. graphSnapshot: "mem:graph:snapshot", + // Per-session fingerprint of the observation set last sent to + // mem::graph-extract on session stop; used to skip re-extraction when + // the session stops again without new observations. + graphExtractState: "mem:graph:extract-state", // #814 v2: targeted-lookup indexes so graph-extract never enumerates // the full nodes/edges scope. Each entry is a single small kv.get, // bounded payload — works at 75K+ nodes where kv.list would block diff --git a/src/triggers/events.ts b/src/triggers/events.ts index e38b58db4d..001c35b145 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -4,6 +4,7 @@ import { KV, STREAM } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { isReflectEnabled } from "../functions/slots.js"; import { isGraphExtractionEnabled } from "../config.js"; +import { computeInputFingerprint } from "../functions/input-fingerprint.js"; import { logger } from "../logger.js"; export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { @@ -67,11 +68,32 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { ); const compressed = observations.filter((o) => o.title); if (compressed.length > 0) { - sdk.trigger({ - function_id: "mem::graph-extract", - payload: { observations: compressed }, - action: TriggerAction.Void(), - }); + // Sessions that stop repeatedly without new observations (e.g. + // heartbeat loops) would re-extract the identical set every time; + // skip when the fingerprint of the set is unchanged. + const fingerprint = computeInputFingerprint(compressed); + const prev = await kv + .get<{ fingerprint: string }>( + KV.graphExtractState, + data.sessionId, + ) + .catch(() => null); + if (prev && prev.fingerprint === fingerprint) { + logger.info("Graph extraction skipped — input unchanged", { + sessionId: data.sessionId, + observationCount: compressed.length, + }); + } else { + await kv.set(KV.graphExtractState, data.sessionId, { + fingerprint, + at: new Date().toISOString(), + }); + sdk.trigger({ + function_id: "mem::graph-extract", + payload: { observations: compressed }, + action: TriggerAction.Void(), + }); + } } } catch (err) { logger.warn("graph-extract trigger failed", { diff --git a/test/session-end-triggers-graph.test.ts b/test/session-end-triggers-graph.test.ts index 22a780432c..869a2856eb 100644 --- a/test/session-end-triggers-graph.test.ts +++ b/test/session-end-triggers-graph.test.ts @@ -91,3 +91,28 @@ describe("agentmemory status no longer depends on /export (#666)", () => { expect(cli).toMatch(/memoriesRes\?\.latestCount\s*\?\?\s*memoriesRes\?\.total/); }); }); + +// Sessions that stop repeatedly without new observations (e.g. heartbeat +// loops) used to re-extract the identical observation set on every stop. +// The handler now fingerprints the set and skips when unchanged. +describe("event::session::stopped graph-extract dedup", () => { + const events = readFileSync("src/triggers/events.ts", "utf-8"); + + it("fingerprints the compressed set before triggering mem::graph-extract", () => { + expect(events).toMatch( + /const fingerprint = computeInputFingerprint\(compressed\);[\s\S]*?function_id:\s*"mem::graph-extract"/, + ); + }); + + it("skips extraction when the stored fingerprint matches", () => { + expect(events).toMatch( + /prev\.fingerprint === fingerprint[\s\S]*?Graph extraction skipped/, + ); + }); + + it("persists the fingerprint under KV.graphExtractState before triggering", () => { + expect(events).toMatch( + /kv\.set\(KV\.graphExtractState,\s*data\.sessionId,\s*\{\s*fingerprint,[\s\S]*?\}\);[\s\S]*?function_id:\s*"mem::graph-extract"/, + ); + }); +}); From aa5fe6f96e0accb297e4e343a7dec0146192b090 Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Thu, 30 Jul 2026 10:51:12 +0800 Subject: [PATCH 4/6] fix(reflect): read graph nodes from snapshot instead of full kv.list `kv.list(KV.graphNodes)` stopped completing once mem:graph:nodes.bin grew past 100 MiB (26K nodes, 3.5M sourceObservationIds). The failure was swallowed by `.catch(() => [])`, so graph clustering silently returned no clusters and reflect fell back to lexical Jaccard clustering, which built a cluster matching 6497 facts and a ~400k-token prompt. Every run then exceeded the 180s invocation timeout, disabling insight generation. Reflect now reads the #814 top-degree snapshot via a single kv.get, and cluster inputs are capped at 50 items per category so the fallback path can no longer produce an unbounded prompt or unbounded sourceMemoryIds. The remaining kv.list calls log a warning instead of degrading silently. Verified in production: 42s, usedFallback=false, 5 new insights, where every prior run timed out at 180s. AI-Agent: Claude Code AI-Session-IDs: 8affe40e-9240-4076-96a8-c71d01a59bdc --- src/functions/graph.ts | 4 ++- src/functions/reflect.ts | 67 ++++++++++++++++++++++++++++++++------- test/reflect.test.ts | 68 ++++++++++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index bb0d727224..fb1ed41830 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -76,7 +76,9 @@ function emptySnapshot(): GraphSnapshot { }; } -async function readSnapshot(kv: StateKV): Promise { +export async function readSnapshot( + kv: StateKV, +): Promise { try { const snap = await kv.get(KV.graphSnapshot, SNAPSHOT_KEY); if (snap && typeof snap === "object" && snap.version === 1) { diff --git a/src/functions/reflect.ts b/src/functions/reflect.ts index 3a2af82934..6726c9be14 100644 --- a/src/functions/reflect.ts +++ b/src/functions/reflect.ts @@ -11,6 +11,8 @@ import type { MemoryProvider, } from "../types.js"; import { recordAudit } from "./audit.js"; +import { readSnapshot } from "./graph.js"; +import { logger } from "../logger.js"; import { REFLECT_SYSTEM, buildReflectPrompt } from "../prompts/reflect.js"; interface ConceptCluster { @@ -170,16 +172,46 @@ export function registerReflectFunctions( const maxClusters = Math.min(data?.maxClusters ?? 10, 20); const maxInsightsPerCluster = 5; const maxTotal = 50; + const maxItemsPerCluster = 50; - const [graphNodes, graphEdges, semanticMemories, lessons, crystals] = + // 讀不到就退化成 fallback 分群,但不能靜默:payload 一旦超過引擎上限 + // 就會在這裡失敗,而 fallback 的 prompt 成本高出兩個數量級。 + const listOrEmpty = async (scope: string): Promise => { + try { + return await kv.list(scope); + } catch (error) { + logger.warn("reflect: kv.list failed, degrading to empty set", { + scope, + error: error instanceof Error ? error.message : String(error), + }); + return []; + } + }; + + // #814 v2: graph nodes come from the precomputed top-degree + // snapshot, never from `kv.list(KV.graphNodes)`. At 26K + // nodes that list burns the entire invocation budget, so reflect + // timed out before it could emit anything. The snapshot is capped + // at the top-degree subgraph, which is far more than the at-most-20 + // clusters this function builds. No snapshot => empty node set => + // the existing lexical fallback below. + const [snapshot, graphEdges, semanticMemories, lessons, crystals] = await Promise.all([ - kv.list(KV.graphNodes).catch(() => []), - kv.list(KV.graphEdges).catch(() => []), - kv.list(KV.semantic).catch(() => []), - kv.list(KV.lessons).catch(() => []), - kv.list(KV.crystals).catch(() => []), + readSnapshot(kv), + listOrEmpty(KV.graphEdges), + listOrEmpty(KV.semantic), + listOrEmpty(KV.lessons), + listOrEmpty(KV.crystals), ]); + if (!snapshot) { + logger.warn( + "reflect: no graph snapshot, degrading to lexical clustering", + { scope: KV.graphSnapshot }, + ); + } + const graphNodes: GraphNode[] = snapshot?.topNodes ?? []; + let activeLessons = lessons.filter((l) => !l.deleted); if (data?.project) { activeLessons = activeLessons.filter((l) => l.project === data.project); @@ -237,20 +269,31 @@ export function registerReflectFunctions( continue; } + // 詞彙 fallback 的 cluster 可以匹配到數千筆 facts,未截斷的 prompt 會 + // 撐到數十萬 token 並超過引擎的 invocation timeout。取信心最高的前段即可。 + const topByConfidence = (items: T[]) => + [...items] + .sort((a, b) => b.confidence - a.confidence) + .slice(0, maxItemsPerCluster); + + const selectedFacts = topByConfidence(clusterFacts); + const selectedLessons = topByConfidence(clusterLessons); + const selectedCrystals = clusterCrystals.slice(0, maxItemsPerCluster); + const cluster: ConceptCluster = { concepts: conceptNames, - facts: clusterFacts.map((f) => ({ + facts: selectedFacts.map((f) => ({ fact: f.fact, confidence: f.confidence, })), - lessons: clusterLessons.map((l) => ({ + lessons: selectedLessons.map((l) => ({ content: l.content, confidence: l.confidence, })), - crystalNarratives: clusterCrystals.map((c) => c.narrative), - factIds: clusterFacts.map((f) => f.id), - lessonIds: clusterLessons.map((l) => l.id), - crystalIds: clusterCrystals.map((c) => c.id), + crystalNarratives: selectedCrystals.map((c) => c.narrative), + factIds: selectedFacts.map((f) => f.id), + lessonIds: selectedLessons.map((l) => l.id), + crystalIds: selectedCrystals.map((c) => c.id), }; try { diff --git a/test/reflect.test.ts b/test/reflect.test.ts index 7ea163d593..4ba8d6e783 100644 --- a/test/reflect.test.ts +++ b/test/reflect.test.ts @@ -57,6 +57,28 @@ function makeConceptNode(name: string): GraphNode { }; } +// Reflect reads graph nodes from the #814 top-degree snapshot, not from +// the mem:graph:nodes scope, so tests seed the snapshot. +async function seedSnapshot( + kv: ReturnType, + nodes: GraphNode[], +): Promise { + await kv.set("mem:graph:snapshot", "current", { + version: 1, + topNodes: nodes, + topEdges: [], + topDegrees: {}, + stats: { + totalNodes: nodes.length, + totalEdges: 0, + nodesByType: {}, + edgesByType: {}, + }, + updatedAt: "2026-04-01T00:00:00Z", + dirty: false, + }); +} + function makeEdge(src: string, tgt: string): GraphEdge { return { id: `edge_${src}_${tgt}`, @@ -151,9 +173,11 @@ describe("Reflect", () => { }); it("synthesizes insights from graph concept clusters", async () => { - await kv.set("mem:graph:nodes", "node_security", makeConceptNode("security")); - await kv.set("mem:graph:nodes", "node_validation", makeConceptNode("validation")); - await kv.set("mem:graph:nodes", "node_testing", makeConceptNode("testing")); + await seedSnapshot(kv, [ + makeConceptNode("security"), + makeConceptNode("validation"), + makeConceptNode("testing"), + ]); await kv.set("mem:graph:edges", "edge_1", makeEdge("security", "validation")); await kv.set("mem:graph:edges", "edge_2", makeEdge("security", "testing")); @@ -178,8 +202,10 @@ describe("Reflect", () => { }); it("skips clusters with fewer than 3 supporting items", async () => { - await kv.set("mem:graph:nodes", "node_sparse", makeConceptNode("sparse")); - await kv.set("mem:graph:nodes", "node_topic", makeConceptNode("topic")); + await seedSnapshot(kv, [ + makeConceptNode("sparse"), + makeConceptNode("topic"), + ]); await kv.set("mem:graph:edges", "edge_1", makeEdge("sparse", "topic")); await kv.set("mem:semantic", "sem_1", makeSemantic("One sparse fact")); @@ -194,8 +220,10 @@ describe("Reflect", () => { }); it("deduplicates insights by fingerprint", async () => { - await kv.set("mem:graph:nodes", "node_security", makeConceptNode("security")); - await kv.set("mem:graph:nodes", "node_validation", makeConceptNode("validation")); + await seedSnapshot(kv, [ + makeConceptNode("security"), + makeConceptNode("validation"), + ]); await kv.set("mem:graph:edges", "edge_1", makeEdge("security", "validation")); await kv.set("mem:semantic", "sem_1", makeSemantic("Always validate security inputs")); await kv.set("mem:semantic", "sem_2", makeSemantic("Testing improves security coverage")); @@ -233,11 +261,33 @@ describe("Reflect", () => { expect(result.usedFallback).toBe(true); }); + it("never enumerates the graph node scope", async () => { + await kv.set("mem:graph:nodes", "node_security", makeConceptNode("security")); + await kv.set("mem:graph:nodes", "node_validation", makeConceptNode("validation")); + await kv.set("mem:graph:edges", "edge_1", makeEdge("security", "validation")); + await kv.set("mem:semantic", "sem_1", makeSemantic("Always validate security inputs")); + await kv.set("mem:semantic", "sem_2", makeSemantic("Testing improves security coverage")); + await kv.set("mem:semantic", "sem_3", makeSemantic("Validation prevents injection")); + + const listSpy = vi.spyOn(kv, "list"); + + const result = (await sdk.trigger("mem::reflect", {})) as { + usedFallback: boolean; + }; + + expect(listSpy.mock.calls.map((c) => c[0])).not.toContain( + "mem:graph:nodes", + ); + expect(result.usedFallback).toBe(true); + }); + it("handles LLM failure gracefully", async () => { provider.summarize.mockRejectedValue(new Error("LLM timeout")); - await kv.set("mem:graph:nodes", "node_a", makeConceptNode("concept_a")); - await kv.set("mem:graph:nodes", "node_b", makeConceptNode("concept_b")); + await seedSnapshot(kv, [ + makeConceptNode("concept_a"), + makeConceptNode("concept_b"), + ]); await kv.set("mem:graph:edges", "edge_1", makeEdge("concept_a", "concept_b")); await kv.set("mem:semantic", "sem_1", makeSemantic("fact about concept_a")); await kv.set("mem:semantic", "sem_2", makeSemantic("fact about concept_b")); From f55201ebc66212ce9ec62508c22398b23e1ad4d6 Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Thu, 30 Jul 2026 11:41:43 +0800 Subject: [PATCH 5/6] fix(health): measure heap usage against the ceiling, not the committed heap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit memPercent divided heapUsed by heapTotal, but heapTotal is the heap V8 has committed so far, which it deliberately keeps near-full and grows on demand. A healthy process therefore reports ~100%, and since the RSS floor (512MB) is trivially cleared by any real workload, the server pinned itself at status=critical and /agentmemory/health returned 503 indefinitely. Observed on the Oracle host: 588MB used of a 603MB committed heap = 97%, while --max-old-space-size=6144 puts real utilisation under 10%. That makes health useless as a liveness signal — a watchdog keyed on it would restart a perfectly healthy server forever. The snapshot now carries heapLimit from v8.getHeapStatistics(), and the ratio uses it when present. heapLimit is optional so snapshots persisted before this change still evaluate against heapTotal as they did before. The RSS floor added for issue #158 was a band-aid over this same defect; it is left in place as a second guard. AI-Agent: Claude Code AI-Session-IDs: 8affe40e-9240-4076-96a8-c71d01a59bdc --- src/health/monitor.ts | 2 ++ src/health/thresholds.ts | 12 +++++++++--- src/types.ts | 2 ++ test/health-thresholds.test.ts | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 953c94a0f4..0f45903920 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -1,3 +1,4 @@ +import v8 from "node:v8"; import type { ISdk } from "iii-sdk"; import type { HealthSnapshot } from "../types.js"; import type { StateKV } from "../state/kv.js"; @@ -69,6 +70,7 @@ export function registerHealthMonitor( memory: { heapUsed: mem.heapUsed, heapTotal: mem.heapTotal, + heapLimit: v8.getHeapStatistics().heap_size_limit, rss: mem.rss, external: mem.external, }, diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad16..087b0e4126 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -59,10 +59,16 @@ export function evaluateHealth( degraded = true; } + // heapTotal is the heap V8 has *committed*, which it keeps near-full by design + // and grows on demand — measuring against it reports ~100% on a perfectly + // healthy process. Measure against the real ceiling (--max-old-space-size) + // when the snapshot carries it; older persisted snapshots do not. + const heapCeiling = + snapshot.memory.heapLimit && snapshot.memory.heapLimit > 0 + ? snapshot.memory.heapLimit + : snapshot.memory.heapTotal; const memPercent = - snapshot.memory.heapTotal > 0 - ? (snapshot.memory.heapUsed / snapshot.memory.heapTotal) * 100 - : 0; + heapCeiling > 0 ? (snapshot.memory.heapUsed / heapCeiling) * 100 : 0; const rss = snapshot.memory.rss ?? 0; const rssAboveFloor = rss >= cfg.memoryRssFloorBytes; const memMb = Math.round(rss / (1024 * 1024)); diff --git a/src/types.ts b/src/types.ts index 5cf5d9d2d1..6604204bd7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -208,6 +208,8 @@ export interface HealthSnapshot { memory: { heapUsed: number; heapTotal: number; + /** V8 heap ceiling (--max-old-space-size). Absent on snapshots persisted before it was collected. */ + heapLimit?: number; rss: number; external: number; }; diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index 6f918a9174..e9de8d7746 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -80,6 +80,38 @@ describe("evaluateHealth memory severity", () => { expect(alerts.some((a) => a.startsWith("memory_warn_"))).toBe(true); }); + it("stays healthy when the committed heap is full but far below the ceiling", () => { + // Observed on the Oracle host: 588MB used of a 603MB committed heap reads as + // 97%, but --max-old-space-size=6144 means real utilisation is under 10%. + const s = snap({ + memory: { + heapUsed: 588 * 1024 * 1024, + heapTotal: 603 * 1024 * 1024, + heapLimit: 6144 * 1024 * 1024, + rss: 1608 * 1024 * 1024, + external: 0, + }, + }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("healthy"); + expect(alerts).toEqual([]); + }); + + it("goes critical when usage approaches the heap ceiling", () => { + const s = snap({ + memory: { + heapUsed: 5900 * 1024 * 1024, + heapTotal: 5950 * 1024 * 1024, + heapLimit: 6144 * 1024 * 1024, + rss: 6500 * 1024 * 1024, + external: 0, + }, + }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(true); + }); + it("respects caller-supplied memoryRssFloorBytes", () => { const s = snap({ memory: { From 947593717baae89786b9ecc63dc702585941cb9e Mon Sep 17 00:00:00 2001 From: Yician Wang Date: Thu, 30 Jul 2026 11:42:17 +0800 Subject: [PATCH 6/6] docs(changelog): record the reflect and health fixes under Unreleased AI-Agent: Claude Code AI-Session-IDs: 8affe40e-9240-4076-96a8-c71d01a59bdc --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5188fa98..1ed07eda01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] +### Fixed + +- **`mem::reflect` no longer enumerates the graph node scope.** `kv.list(KV.graphNodes)` stopped completing once `mem:graph:nodes` grew past ~100 MB, and the failure was swallowed by a silent `.catch(() => [])`. Graph clustering then returned nothing, reflect fell back to lexical clustering, and a cluster matching thousands of facts produced a prompt large enough to exceed the 180s invocation timeout — every run, silently disabling insight generation. Reflect now reads the top-degree graph snapshot (the same fast path graph extraction already used), cluster inputs are capped at 50 items per category, and the remaining `kv.list` calls log a warning instead of degrading silently. +- **Health no longer pins itself to `critical` on a healthy process.** Memory severity divided `heapUsed` by `heapTotal`, but `heapTotal` is the heap V8 has committed so far, which it deliberately keeps near-full and grows on demand. Any process past the RSS floor therefore reported ~100% and `/agentmemory/health` returned 503 indefinitely, making it useless as a liveness signal. The snapshot now carries `heapLimit` from `v8.getHeapStatistics()` and the ratio is measured against that ceiling; snapshots persisted without it still evaluate as before. + ## [0.9.27] — 2026-06-07 Wave release closing several breaking regressions reported against v0.9.26, plus an agent-scope isolation security fix, an iii version-pin audit fix, and a benchmark scorecard correction. No breaking changes; drop-in upgrade.