From 44e5372028dc80591eb9496f1024443296eb507f Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 05:52:46 +0800 Subject: [PATCH 01/27] fix(health): measure heap severity against the V8 limit evaluateHealth divided heapUsed by heapTotal. heapTotal is the heap V8 has committed so far, not the ceiling it may grow to, and V8 sizes it to demand, so a healthy busy process sits near 100% of it more or less permanently. Since heapTotal <= heap_size_limit always holds, the ratio could only ever over-report severity, never under-report it. A live deployment reported memory_critical_97% while using 6% of its 6192 MB heap limit. That made GET /agentmemory/health return 503 on a service whose container was at 3.0 GB of an 8 GB limit with the circuit breaker closed. The snapshot now carries heapSizeLimit, measured in monitor.ts alongside process.memoryUsage(). evaluateHealth measures against it when present and falls back to heapTotal when absent, so existing callers and any persisted snapshot keep their current behaviour. The 512 MB RSS floor from #158 stays as-is. It suppressed the alert for small processes but left the ratio wrong, so the report returned for any process above the floor. Refs: #1223 Signed-off-by: Omar Gerardo --- src/health/monitor.ts | 2 + src/health/thresholds.ts | 11 +++-- src/types.ts | 2 + test/health-thresholds.test.ts | 77 ++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 953c94a0f4..1de815321c 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -1,3 +1,4 @@ +import { getHeapStatistics } from "node:v8"; import type { ISdk } from "iii-sdk"; import type { HealthSnapshot } from "../types.js"; import type { StateKV } from "../state/kv.js"; @@ -71,6 +72,7 @@ export function registerHealthMonitor( heapTotal: mem.heapTotal, rss: mem.rss, external: mem.external, + heapSizeLimit: getHeapStatistics().heap_size_limit, }, cpu: { userMicros: currentCpu.user, diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad16..e1eab3a4f5 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -59,10 +59,15 @@ export function evaluateHealth( degraded = true; } + // heapTotal is what V8 has committed so far, not what it may grow to, and V8 + // sizes it to demand — so a healthy busy process sits near 100% of it + // permanently. Measure against heap_size_limit when the snapshot carries it. + const heapCeiling = + snapshot.memory.heapSizeLimit && snapshot.memory.heapSizeLimit > 0 + ? snapshot.memory.heapSizeLimit + : snapshot.memory.heapTotal; const memPercent = - snapshot.memory.heapTotal > 0 - ? (snapshot.memory.heapUsed / snapshot.memory.heapTotal) * 100 - : 0; + heapCeiling > 0 ? (snapshot.memory.heapUsed / heapCeiling) * 100 : 0; const rss = snapshot.memory.rss ?? 0; const rssAboveFloor = rss >= cfg.memoryRssFloorBytes; const memMb = Math.round(rss / (1024 * 1024)); diff --git a/src/types.ts b/src/types.ts index 1118b3f999..b85caf23a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -226,6 +226,8 @@ export interface HealthSnapshot { heapTotal: number; rss: number; external: number; + /** v8.getHeapStatistics().heap_size_limit — the ceiling V8 may grow to. */ + heapSizeLimit?: number; }; cpu: { userMicros: number; systemMicros: number; percent: number }; eventLoopLagMs: number; diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index 6f918a9174..cb80a991b1 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -95,3 +95,80 @@ describe("evaluateHealth memory severity", () => { expect(strict.status).toBe("healthy"); }); }); + +describe("evaluateHealth memory severity — denominator", () => { + const LIMIT_6192MB = 6192 * 1024 * 1024; + + it("stays healthy when a busy process fills its committed heap but sits far below the V8 limit", () => { + // Captured from a live deployment: this process reported + // memory_critical_97% while using 6% of the heap it may grow to. + const s = snap({ + memory: { + heapUsed: 390_771_624, + heapTotal: 404_930_560, + rss: 584_327_168, + external: 6_566_392, + heapSizeLimit: LIMIT_6192MB, + }, + }); + + const { status, alerts } = evaluateHealth(s); + + expect(status).toBe("healthy"); + expect(alerts.find((a) => a.startsWith("memory_critical_"))).toBeUndefined(); + expect(alerts.find((a) => a.startsWith("memory_warn_"))).toBeUndefined(); + }); + + it("reports the percentage against the limit, not against the committed heap", () => { + const s = snap({ + memory: { + heapUsed: 390_771_624, + heapTotal: 404_930_560, + rss: 584_327_168, + external: 0, + heapSizeLimit: LIMIT_6192MB, + }, + }); + + // Widen the band so the alert fires as a warn either way and the rendered + // percentage is the only thing under test. + const { alerts } = evaluateHealth(s, { + memoryWarnPercent: 1, + memoryCriticalPercent: 99, + }); + const warn = alerts.find((a) => a.startsWith("memory_warn_")); + + // 390771624 / 6492782592 = 6%, not 390771624 / 404930560 = 97%. + expect(warn).toBe("memory_warn_6%_rss557mb"); + }); + + it("still goes critical when the heap genuinely approaches the V8 limit", () => { + const s = snap({ + memory: { + heapUsed: 6_200_000_000, + heapTotal: 6_300_000_000, + rss: 6_800_000_000, + external: 0, + heapSizeLimit: LIMIT_6192MB, + }, + }); + + const { status, alerts } = evaluateHealth(s); + + expect(status).toBe("critical"); + expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(true); + }); + + it("falls back to the committed heap when heapSizeLimit is absent", () => { + const s = snap({ + memory: { + heapUsed: 970 * 1024 * 1024, + heapTotal: 1000 * 1024 * 1024, + rss: 1100 * 1024 * 1024, + external: 0, + }, + }); + + expect(evaluateHealth(s).status).toBe("critical"); + }); +}); From 357ba53d76310848172fab9eca41e554c93dba99 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 05:53:01 +0800 Subject: [PATCH 02/27] fix(viewer): draw the heap gauge against the V8 limit The dashboard recomputed heapUsed / heapTotal client-side, so it carried the same over-reporting as the health thresholds did. On a process using 6% of its heap limit the gauge still rendered red and read "438 / 484 MB", contradicting the status the API now reports. It reads heapSizeLimit from the snapshot when present and falls back to heapTotal, and the label names the same ceiling the bar measures against. Refs: #1223 Signed-off-by: Omar Gerardo --- src/viewer/index.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/viewer/index.html b/src/viewer/index.html index de47f4f5f9..afb39d164c 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -1579,10 +1579,12 @@

agentmemory

var heapUsed = Math.round((snap.memory.heapUsed || 0) / 1024 / 1024); var heapTotal = Math.round((snap.memory.heapTotal || 0) / 1024 / 1024); var rss = Math.round((snap.memory.rss || 0) / 1024 / 1024); - var heapPct = heapTotal > 0 ? Math.round((heapUsed / heapTotal) * 100) : 0; + var heapLimit = Math.round((snap.memory.heapSizeLimit || 0) / 1024 / 1024); + var heapCeiling = heapLimit > 0 ? heapLimit : heapTotal; + var heapPct = heapCeiling > 0 ? Math.round((heapUsed / heapCeiling) * 100) : 0; var rssAboveFloor = rss >= 512; var heapColor = (heapPct > 80 && rssAboveFloor) ? 'var(--red)' : (heapPct > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; - html += '
Heap
' + heapUsed + ' / ' + heapTotal + ' MB
'; + html += '
Heap
' + heapUsed + ' / ' + heapCeiling + ' MB
'; html += '
RSS
' + rss + ' MB
'; if (snap.memory.external) { var ext = Math.round(snap.memory.external / 1024 / 1024); From 36c862435423ad69f14f2f6aa7bcd70ac2994890 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 06:15:48 +0800 Subject: [PATCH 03/27] fix(viewer): compute the heap percentage from raw bytes The gauge rounded both operands to MB before dividing, while evaluateHealth divides the raw byte values. Near a threshold boundary the two could land on different sides of it, which defeats the point of having the gauge mirror the health status. The percentage now comes off the byte values, and the MB rounding applies only to the label the gauge displays. Refs: #1223 Signed-off-by: Omar Gerardo --- src/viewer/index.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/viewer/index.html b/src/viewer/index.html index afb39d164c..970582a874 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -1579,9 +1579,12 @@

agentmemory

var heapUsed = Math.round((snap.memory.heapUsed || 0) / 1024 / 1024); var heapTotal = Math.round((snap.memory.heapTotal || 0) / 1024 / 1024); var rss = Math.round((snap.memory.rss || 0) / 1024 / 1024); - var heapLimit = Math.round((snap.memory.heapSizeLimit || 0) / 1024 / 1024); - var heapCeiling = heapLimit > 0 ? heapLimit : heapTotal; - var heapPct = heapCeiling > 0 ? Math.round((heapUsed / heapCeiling) * 100) : 0; + var limitBytes = snap.memory.heapSizeLimit || 0; + var ceilingBytes = limitBytes > 0 ? limitBytes : (snap.memory.heapTotal || 0); + var heapCeiling = Math.round(ceilingBytes / 1024 / 1024); + // Percentage off raw bytes so the gauge agrees with evaluateHealth, + // which never sees the MB rounding used for the label. + var heapPct = ceilingBytes > 0 ? Math.round(((snap.memory.heapUsed || 0) / ceilingBytes) * 100) : 0; var rssAboveFloor = rss >= 512; var heapColor = (heapPct > 80 && rssAboveFloor) ? 'var(--red)' : (heapPct > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; html += '
Heap
' + heapUsed + ' / ' + heapCeiling + ' MB
'; From e87a7b64074ccc454364efedac321c7616178a7b Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 06:22:26 +0800 Subject: [PATCH 04/27] test(viewer): cover the heap gauge ceiling and precision The gauge changes had no coverage, while viewer-graph-cooldown and viewer-memories-sort already assert viewer behaviour the same way. This follows that pattern: read src/viewer/index.html and assert the shape of the emitted JS. Three cases. The gauge measures against heapSizeLimit and falls back to heapTotal, the percentage divides raw byte values rather than the MB-rounded label numbers, and the rounding is applied only where the label is built. Each one fails against the previous implementation. Refs: #1223 Signed-off-by: Omar Gerardo --- test/viewer-heap-gauge.test.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 test/viewer-heap-gauge.test.ts diff --git a/test/viewer-heap-gauge.test.ts b/test/viewer-heap-gauge.test.ts new file mode 100644 index 0000000000..c50dd93240 --- /dev/null +++ b/test/viewer-heap-gauge.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; + +// The dashboard used to recompute heapUsed / heapTotal client-side, which is +// the same over-reporting the health thresholds had: heapTotal is what V8 has +// committed, not what it may grow to. The gauge now measures against +// heapSizeLimit, and it divides raw byte values so it cannot disagree with +// evaluateHealth over a rounding step. +describe("viewer heap gauge", () => { + const viewer = readFileSync("src/viewer/index.html", "utf-8"); + + it("measures against the V8 heap limit, falling back to heapTotal", () => { + expect(viewer).toMatch(/limitBytes\s*=\s*snap\.memory\.heapSizeLimit\s*\|\|\s*0/); + expect(viewer).toMatch( + /ceilingBytes\s*=\s*limitBytes\s*>\s*0\s*\?\s*limitBytes\s*:\s*\(snap\.memory\.heapTotal\s*\|\|\s*0\)/, + ); + }); + + it("computes the percentage from raw bytes, not the MB-rounded label values", () => { + expect(viewer).toMatch( + /heapPct\s*=\s*ceilingBytes\s*>\s*0\s*\?\s*Math\.round\(\(\(snap\.memory\.heapUsed\s*\|\|\s*0\)\s*\/\s*ceilingBytes\)\s*\*\s*100\)/, + ); + // The old form divided two already-rounded MB numbers. + expect(viewer).not.toMatch(/heapPct\s*=\s*heapTotal\s*>\s*0\s*\?\s*Math\.round\(\(heapUsed\s*\/\s*heapTotal\)/); + }); + + it("rounds only for the displayed label", () => { + expect(viewer).toMatch(/heapCeiling\s*=\s*Math\.round\(ceilingBytes\s*\/\s*1024\s*\/\s*1024\)/); + expect(viewer).toMatch(/\+\s*heapUsed\s*\+\s*' \/ '\s*\+\s*heapCeiling\s*\+\s*' MB/); + }); +}); From ebdf96143f9105f7af11430fc448254a356df764 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 09:54:01 +0800 Subject: [PATCH 05/27] test(viewer): note why the gauge test asserts on source Review asked why these assert on the emitted JS instead of running the gauge. The answer was not visible from the file: the viewer ships as a single HTML file with inline JS, so there is nothing to import and execute, and the two existing viewer tests solve it the same way. Refs: #1223 Signed-off-by: Omar Gerardo --- test/viewer-heap-gauge.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/viewer-heap-gauge.test.ts b/test/viewer-heap-gauge.test.ts index c50dd93240..5a2759850b 100644 --- a/test/viewer-heap-gauge.test.ts +++ b/test/viewer-heap-gauge.test.ts @@ -6,6 +6,10 @@ import { readFileSync } from "node:fs"; // committed, not what it may grow to. The gauge now measures against // heapSizeLimit, and it divides raw byte values so it cannot disagree with // evaluateHealth over a rounding step. +// +// Asserting on the emitted source rather than running the gauge follows +// viewer-graph-cooldown and viewer-memories-sort: the viewer ships as one +// HTML file with inline JS, so there is no module to import and execute. describe("viewer heap gauge", () => { const viewer = readFileSync("src/viewer/index.html", "utf-8"); From 5843b18f01a2514276798bd30c4355e10d1bdb9a Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 09:58:57 +0800 Subject: [PATCH 06/27] fix(viewer): pick the gauge colour on the unrounded percentage The colour was chosen from the rounded display value while evaluateHealth compares the raw one, so the two disagreed inside a whole percentage point. At 80.4% health warns, but a rounded 80 failed the gauge's `> 80` test and left the bar on the lower colour. The percentage is now kept unrounded for the comparison and rounded only for the label. Same shape as the surrounding fix: round at the display edge, never before a threshold test. Refs: #1223 Signed-off-by: Omar Gerardo --- src/viewer/index.html | 5 +++-- test/viewer-heap-gauge.test.ts | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/viewer/index.html b/src/viewer/index.html index 970582a874..0a2982584c 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -1584,9 +1584,10 @@

agentmemory

var heapCeiling = Math.round(ceilingBytes / 1024 / 1024); // Percentage off raw bytes so the gauge agrees with evaluateHealth, // which never sees the MB rounding used for the label. - var heapPct = ceilingBytes > 0 ? Math.round(((snap.memory.heapUsed || 0) / ceilingBytes) * 100) : 0; + var heapPercent = ceilingBytes > 0 ? ((snap.memory.heapUsed || 0) / ceilingBytes) * 100 : 0; + var heapPct = Math.round(heapPercent); var rssAboveFloor = rss >= 512; - var heapColor = (heapPct > 80 && rssAboveFloor) ? 'var(--red)' : (heapPct > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; + var heapColor = (heapPercent > 80 && rssAboveFloor) ? 'var(--red)' : (heapPercent > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; html += '
Heap
' + heapUsed + ' / ' + heapCeiling + ' MB
'; html += '
RSS
' + rss + ' MB
'; if (snap.memory.external) { diff --git a/test/viewer-heap-gauge.test.ts b/test/viewer-heap-gauge.test.ts index 5a2759850b..b99ee4907a 100644 --- a/test/viewer-heap-gauge.test.ts +++ b/test/viewer-heap-gauge.test.ts @@ -22,12 +22,21 @@ describe("viewer heap gauge", () => { it("computes the percentage from raw bytes, not the MB-rounded label values", () => { expect(viewer).toMatch( - /heapPct\s*=\s*ceilingBytes\s*>\s*0\s*\?\s*Math\.round\(\(\(snap\.memory\.heapUsed\s*\|\|\s*0\)\s*\/\s*ceilingBytes\)\s*\*\s*100\)/, + /heapPercent\s*=\s*ceilingBytes\s*>\s*0\s*\?\s*\(\(snap\.memory\.heapUsed\s*\|\|\s*0\)\s*\/\s*ceilingBytes\)\s*\*\s*100/, ); // The old form divided two already-rounded MB numbers. expect(viewer).not.toMatch(/heapPct\s*=\s*heapTotal\s*>\s*0\s*\?\s*Math\.round\(\(heapUsed\s*\/\s*heapTotal\)/); }); + it("picks the gauge colour on the unrounded percentage", () => { + // Rounding first put the gauge a whole point out of step with + // evaluateHealth, which compares the raw value: at 80.4% health warns + // while a rounded 80 left the bar on the lower colour. + expect(viewer).toMatch(/heapPct\s*=\s*Math\.round\(heapPercent\)/); + expect(viewer).toMatch(/heapColor\s*=\s*\(heapPercent\s*>\s*80\s*&&\s*rssAboveFloor\)/); + expect(viewer).toMatch(/\(heapPercent\s*>\s*60\s*&&\s*rssAboveFloor\)/); + }); + it("rounds only for the displayed label", () => { expect(viewer).toMatch(/heapCeiling\s*=\s*Math\.round\(ceilingBytes\s*\/\s*1024\s*\/\s*1024\)/); expect(viewer).toMatch(/\+\s*heapUsed\s*\+\s*' \/ '\s*\+\s*heapCeiling\s*\+\s*' MB/); From 26e03780500ba8574467da3473163ec4574f604d Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Tue, 18 Aug 2026 09:59:10 +0800 Subject: [PATCH 07/27] refactor(types): drop the redundant heapSizeLimit comment The doc comment named the V8 call the field comes from, but monitor.ts already shows that at the only place the field is populated, so the comment restated what the code says. Refs: #1223 Signed-off-by: Omar Gerardo --- src/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index b85caf23a7..d8543c13ab 100644 --- a/src/types.ts +++ b/src/types.ts @@ -226,7 +226,6 @@ export interface HealthSnapshot { heapTotal: number; rss: number; external: number; - /** v8.getHeapStatistics().heap_size_limit — the ceiling V8 may grow to. */ heapSizeLimit?: number; }; cpu: { userMicros: number; systemMicros: number; percent: number }; From 9fee40ba4e7cfd8de5c846f9763667b490c976a4 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 00:45:11 +0800 Subject: [PATCH 08/27] fix(health): act on the KV probe the monitor already collected registerHealthMonitor has always probed the state store every 30s and measured event-loop lag, then written the snapshot to KV and returned. evaluateHealth never read kvConnectivity, so the one signal that reaches the state store was collected and discarded. Treat a failed probe as critical, and add an opt-in escalation that exits the process after 10 consecutive failures so the platform can restart it. The escalation gates on kvConnectivity specifically, not on snapshot.status: evaluateHealth raises critical from five independent conditions, so gating on the aggregate would let a CPU spike during a consolidation pass kill a process that is not wedged. It also refuses to arm until one healthy probe has been seen, which stops a store that is already stalled at boot from escalating on every container start and spending the platform's restart budget. The counter advances before the snapshot persist, not after. That persist races no timeout, and a stalled store parks it for the engine's full invocation timeout, so a counter behind it would never advance during the exact failure this exists to catch. Two source-order assertions pin that ordering, since no test constructs registerHealthMonitor and the call site is otherwise executed by nothing. Threshold is 10 rather than 3 because src/index.ts documents state::set exceeding the SDK's 30s timeout under sustained hook load, and a 5s probe cannot tell a slow store from a dead one. Also race the workers probe. Unraced it inherited the engine's 180s invocation timeout and parked the whole collection upstream of the probe. The connection block in thresholds.ts is now documented as unreachable: iii-sdk's setConnectionState only assigns a private field and emits nothing, so connectionState stays "connected" for the life of the process. --- src/health/monitor.ts | 99 ++++++++++++++++++++++- src/health/thresholds.ts | 18 +++++ src/viewer/index.html | 2 +- test/health-monitor.test.ts | 142 +++++++++++++++++++++++++++++++++ test/health-thresholds.test.ts | 45 +++++++++++ 5 files changed, 301 insertions(+), 5 deletions(-) create mode 100644 test/health-monitor.test.ts diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 1de815321c..6985be3e4f 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -5,10 +5,86 @@ import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { evaluateHealth } from "./thresholds.js"; +export interface EscalationState { + /** Consecutive snapshots whose KV probe failed. Reset by any healthy probe. */ + consecutiveStalls: number; + /** Set once a healthy KV probe has been seen. Nothing escalates before then. */ + armed: boolean; + escalated: boolean; +} + +/** + * Advance the stall counter and report whether the process should exit. + * + * Gated on the KV probe specifically, NOT on `snapshot.status`. `evaluateHealth` + * raises `critical` from five independent conditions (connection, KV, event-loop + * lag, CPU, memory), so gating on the aggregate would let a CPU spike during a + * consolidation pass kill a process that is not wedged at all. + * + * `armed` mirrors the shell watchdog's "wait for a first success" rule. Without + * it, a store that is already stalled at boot escalates on the first minute of + * every container life, which spends the platform's restart budget in minutes + * and ends at a stopped deployment. + * + * Call this BEFORE persisting the snapshot. The persist is not raced against any + * timeout, and a stalled store parks it for the engine's full invocation timeout, + * so a counter behind it would never advance during the failure it exists to + * catch. + */ +export function bumpEscalation( + snapshot: HealthSnapshot, + state: EscalationState, + threshold: number, +): boolean { + const stalled = snapshot.kvConnectivity?.status === "error"; + if (!stalled) { + state.armed = true; + state.consecutiveStalls = 0; + return false; + } + if (!state.armed) return false; + state.consecutiveStalls += 1; + if (state.escalated || state.consecutiveStalls < threshold) return false; + state.escalated = true; + return true; +} + export function registerHealthMonitor( sdk: ISdk, kv: StateKV, ): { stop: () => void } { + const escalationState: EscalationState = { + consecutiveStalls: 0, + armed: false, + escalated: false, + }; + // Default off. Enabling this arms an automatic process-killer, so it stays + // opt-in until external uptime monitoring exists to make a restart loop + // visible. The threshold spans 5 minutes at the 30s collection interval, + // deliberately wider than the write bursts this codebase already documents + // as able to exceed the engine's own 30s timeout (see src/index.ts). + const escalateEnabled = process.env["AGENTMEMORY_HEALTH_ESCALATE"] === "1" || + process.env["AGENTMEMORY_HEALTH_ESCALATE"]?.toLowerCase() === "true"; + const escalateAfter = 10; + + function escalate(alerts: string[]): void { + console.error( + `[agentmemory] health escalation: KV unreachable for ${escalateAfter} consecutive checks (${alerts.join(", ")}); exiting so the platform restarts`, + ); + // Prefer the registered SIGTERM shutdown so the index flushes. That handler + // ends in process.exit(0), so railway.json uses restartPolicyType ALWAYS — + // ON_FAILURE would read a graceful exit as success and never restart. + process.kill(process.pid, "SIGTERM"); + // NOT unref'd. If shutdown parks on the same stalled store, this is the only + // thing that still reaches an exit. + setTimeout(() => { + console.error( + "[agentmemory] health escalation: graceful shutdown did not finish; forcing exit", + ); + process.exit(1); + }, 15_000); + } + let connectionState = "connected"; let prevCpuUsage = process.cpuUsage(); let prevCpuTime = Date.now(); @@ -38,11 +114,21 @@ export function registerHealthMonitor( const eventLoopLagMs = performance.now() - startMark; let workers: HealthSnapshot["workers"] = []; + // Raced, like the KV probe below. Unraced, this inherits the engine's + // invocation timeout (180s), so a dead engine parks the whole collection + // here -- upstream of the probe and of the escalation decision -- and + // detection slips from minutes to tens of minutes. + const WORKERS_PROBE_TIMEOUT = 5000; try { - const result = await sdk.trigger< - unknown, - { workers?: HealthSnapshot["workers"] } - >({ function_id: "engine::workers::list", payload: {} }); + const result = await Promise.race([ + sdk.trigger({ + function_id: "engine::workers::list", + payload: {}, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("timeout")), WORKERS_PROBE_TIMEOUT), + ), + ]); if (result?.workers) workers = result.workers; } catch {} @@ -91,6 +177,11 @@ export function registerHealthMonitor( snapshot.alerts = evaluated.alerts; snapshot.notes = evaluated.notes; + // Decide before the persist below, which races no timeout. + if (escalateEnabled && bumpEscalation(snapshot, escalationState, escalateAfter)) { + escalate(snapshot.alerts); + } + await kv.set(KV.health, "latest", snapshot).catch(() => {}); return snapshot; } diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index e1eab3a4f5..cf5bd11083 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -30,6 +30,12 @@ export function evaluateHealth( let critical = false; let degraded = false; + // NOTE: unreachable in production today. iii-sdk's setConnectionState only + // assigns a private field and emits nothing, so the "connection_state" + // listener in monitor.ts never fires and connectionState stays "connected" + // for the life of the process. Kept because the field is part of the + // snapshot contract and a future SDK may emit it; do not rely on it as a + // liveness signal until it does. if ( snapshot.connectionState === "disconnected" || snapshot.connectionState === "failed" @@ -41,6 +47,18 @@ export function evaluateHealth( degraded = true; } + // The KV probe in collectHealth is the only check that exercises the state + // store end to end (set then get, raced against a 5s timeout). A store that + // stops answering takes the HTTP workers down with it, so a failed probe is + // the earliest reliable signal of that failure and belongs at critical. + // kvConnectivity is optional on the snapshot, and older persisted snapshots + // predate it, so an absent or malformed value must read as "no signal" + // rather than as a failure. + if (snapshot.kvConnectivity?.status === "error") { + alerts.push("kv_probe_failed"); + critical = true; + } + if (snapshot.eventLoopLagMs > cfg.eventLoopLagCriticalMs) { alerts.push( `event_loop_lag_critical_${Math.round(snapshot.eventLoopLagMs)}ms`, diff --git a/src/viewer/index.html b/src/viewer/index.html index 0a2982584c..c532a76e9b 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -1210,7 +1210,7 @@

agentmemory

return 'CPU ' + (m[1] === 'critical' ? 'critically high' : 'elevated') + ': ' + m[2] + '%.'; if ((m = /^event_loop_lag_(warn|critical)_(\d+)ms$/.exec(f))) return 'Event loop ' + (m[1] === 'critical' ? 'severely delayed' : 'delayed') + ': ' + m[2] + ' ms behind. The worker is busy or blocked.'; - if (f === 'connection_reconnecting') + if (f === 'kv_probe_failed') return 'State store unreachable: the health probe could not read back what it wrote. Search and recall are likely stalled.'; if (f === 'connection_reconnecting') return 'Engine connection lost — reconnecting.'; if ((m = /^connection_(.+)$/.exec(f))) return 'Engine connection state: ' + m[1] + '.'; diff --git a/test/health-monitor.test.ts b/test/health-monitor.test.ts new file mode 100644 index 0000000000..14b42e0f6c --- /dev/null +++ b/test/health-monitor.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import type { HealthSnapshot } from "../src/types.js"; +import { bumpEscalation, type EscalationState } from "../src/health/monitor.js"; + +function snap(over: Partial = {}): HealthSnapshot { + return { + connectionState: "connected", + workers: [], + memory: { heapUsed: 0, heapTotal: 1, rss: 0, external: 0 }, + cpu: { userMicros: 0, systemMicros: 0, percent: 0 }, + eventLoopLagMs: 0, + uptimeSeconds: 600, + kvConnectivity: { status: "ok", latencyMs: 1 }, + status: "healthy", + alerts: [], + ...over, + }; +} + +const stalled = () => snap({ kvConnectivity: { status: "error", error: "kv_probe_failed" } }); + +function fresh(): EscalationState { + return { consecutiveStalls: 0, armed: false, escalated: false }; +} + +/** Arm the state the way a healthy first collection would. */ +function armed(): EscalationState { + const s = fresh(); + bumpEscalation(snap(), s, 3); + return s; +} + +describe("bumpEscalation arming", () => { + // Mirrors the shell watchdog's "wait for a first success" rule. Without it a + // store that is already stalled at boot escalates on the first minute of every + // container life, spending the platform's restart budget in minutes. + it("never escalates when the store is stalled from the very first check", () => { + const state = fresh(); + for (let i = 0; i < 50; i++) { + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + } + expect(state.armed).toBe(false); + expect(state.consecutiveStalls).toBe(0); + }); + + it("arms on the first healthy probe", () => { + const state = fresh(); + bumpEscalation(snap(), state, 3); + expect(state.armed).toBe(true); + }); +}); + +describe("bumpEscalation gating", () => { + it("escalates on the Nth consecutive stall, not before", () => { + const state = armed(); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + expect(bumpEscalation(stalled(), state, 3)).toBe(true); + }); + + it("resets the counter on a healthy probe", () => { + const state = armed(); + bumpEscalation(stalled(), state, 3); + bumpEscalation(stalled(), state, 3); + bumpEscalation(snap(), state, 3); + expect(state.consecutiveStalls).toBe(0); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + }); + + it("escalates at most once per process", () => { + const state = armed(); + let fired = 0; + for (let i = 0; i < 20; i++) { + if (bumpEscalation(stalled(), state, 1)) fired++; + } + expect(fired).toBe(1); + }); + + // The gate is the KV probe, not snapshot.status. evaluateHealth raises + // `critical` from five independent conditions, so gating on the aggregate + // would let a CPU spike during a consolidation pass kill a healthy process. + it.each([ + ["cpu", { cpu: { userMicros: 0, systemMicros: 0, percent: 99 } }], + ["event loop lag", { eventLoopLagMs: 5000 }], + ["connection", { connectionState: "disconnected" }], + ["memory", { memory: { heapUsed: 99, heapTotal: 100, rss: 99, external: 0 } }], + ])("never escalates on a %s critical while the KV probe is healthy", (_label, over) => { + const state = armed(); + for (let i = 0; i < 50; i++) { + expect( + bumpEscalation(snap({ ...over, status: "critical" } as Partial), state, 3), + ).toBe(false); + } + expect(state.consecutiveStalls).toBe(0); + }); + + it("treats an absent kvConnectivity as no signal, not as a stall", () => { + const state = armed(); + const s = snap(); + delete s.kvConnectivity; + for (let i = 0; i < 50; i++) { + expect(bumpEscalation(s, state, 3)).toBe(false); + } + expect(state.consecutiveStalls).toBe(0); + }); +}); + +// KTD7. The escalation decision must precede the snapshot persist. `kv.set` +// routes through `sdk.trigger`, whose invocationTimeoutMs is 180000 +// (src/index.ts), so a stalled store parks the persist for three minutes. A +// counter placed behind it would never advance during the exact failure it +// exists to catch. +// +// Asserted on source order because no test constructs registerHealthMonitor, so +// the call site is otherwise executed by nothing. Verified discriminating: this +// passes on the real tree and fails on a tree with the two lines swapped. +// Matches the precedent in test/cli-second-instance-guard.test.ts. +describe("collectHealth statement order", () => { + const src = readFileSync( + fileURLToPath(new URL("../src/health/monitor.ts", import.meta.url)), + "utf8", + ); + + it("decides escalation before the un-raced snapshot persist", () => { + const decide = src.indexOf("bumpEscalation(snapshot, escalationState"); + const persist = src.indexOf('kv.set(KV.health, "latest"'); + expect(decide).toBeGreaterThan(-1); + expect(persist).toBeGreaterThan(-1); + expect(decide).toBeLessThan(persist); + }); + + // The workers probe sits upstream of both. Unraced it inherits the engine's + // 180s invocation timeout and delays detection by tens of minutes. + it("races the workers probe so it cannot park the collection", () => { + const workers = src.indexOf('function_id: "engine::workers::list"'); + expect(workers).toBeGreaterThan(-1); + const before = src.slice(Math.max(0, workers - 400), workers); + expect(before).toContain("Promise.race"); + }); +}); diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index cb80a991b1..bae18455ef 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -172,3 +172,48 @@ describe("evaluateHealth memory severity — denominator", () => { expect(evaluateHealth(s).status).toBe("critical"); }); }); + +describe("evaluateHealth KV connectivity", () => { + it("goes critical when the KV probe fails", () => { + const s = snap({ + kvConnectivity: { status: "error", error: "kv_probe_failed", latencyMs: 5000 }, + }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts).toContain("kv_probe_failed"); + }); + + it("adds no KV alert when the probe succeeds", () => { + const { status, alerts } = evaluateHealth(snap()); + expect(status).toBe("healthy"); + expect(alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); + }); + + // kvConnectivity is optional on HealthSnapshot, and snapshots persisted before + // the field existed still come back from KV, so absence must read as "no + // signal" rather than as a failure. + it("stays healthy when kvConnectivity is absent", () => { + const s = snap(); + delete s.kvConnectivity; + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("healthy"); + expect(alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); + }); + + it("does not throw or alert on a malformed kvConnectivity", () => { + const s = snap({ kvConnectivity: { status: undefined as unknown as string } }); + expect(() => evaluateHealth(s)).not.toThrow(); + expect(evaluateHealth(s).alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); + }); + + it("reports the KV alert alongside other critical signals", () => { + const s = snap({ + kvConnectivity: { status: "error", error: "kv_probe_failed" }, + eventLoopLagMs: 900, + }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts).toContain("kv_probe_failed"); + expect(alerts.find((a) => a.startsWith("event_loop_lag_critical_"))).toBeDefined(); + }); +}); From 8fd1c32670bb628d54ab3475fd14a9e0de612c1f Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 00:45:19 +0800 Subject: [PATCH 09/27] fix(cli): exit when the iii engine dies after startup The engine is spawned detached and already had an exit handler carrying the exit code, the signal, and up to 16KB of its dying stderr. It set startupFailure, logged under vlog, and returned. startupFailure is only ever read on startup paths, so an engine that died while serving was recorded and forgotten. That is the failure this deployment actually hits. The engine owns the REST listener, the stream port, and the state store, so once it dies the surviving node process cannot serve anything, but it stays up reconnecting forever. The platform sees a live container and no HTTP, and reports the service healthy. Confirmed by inspecting a wedged container: no iii process, 3111/3112/49134 all unbound, only the viewer's loopback listener left. Report the death on stderr with its captured output and exit non-zero so a supervisor restarts the container. On by default, unlike the probe-based escalation. This acts on a process-exit event rather than a threshold, so there is no false positive to trade against. Opt out with AGENTMEMORY_EXIT_ON_ENGINE_DEATH=0. The startup grace is deliberately short. It exists so a failed spawn keeps the startup path's clearer message, and a failed spawn surfaces within a second or two. A longer window is a hole in the only mechanism that catches this: an engine dying inside it after startup completed would leave the container up and serving nothing, which is the bug itself. Verified in a container: engine killed at 11s and at 63s both exit 1 with the diagnostic; the opt-out leaves it running. --- src/cli.ts | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/cli.ts b/src/cli.ts index 8bbdda9322..b965782044 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -936,6 +936,7 @@ function spawnEngineBackground( if (!isDocker && typeof child.pid === "number") { writeEnginePidfile(child.pid); } + const spawnedAt = Date.now(); const stderrChunks: Buffer[] = []; let stderrBytes = 0; const MAX_STDERR_CAPTURE = 16 * 1024; @@ -965,12 +966,44 @@ function spawnEngineBackground( } if (!isDocker) clearEnginePidfile(); clearEngineState(); + + // The engine owns the REST listener, the stream port and the state store. + // When it dies the surviving node process cannot serve anything, but it + // stays up reconnecting forever, so the platform sees a healthy container + // and no HTTP. That is the shape of every wedge observed so far. + // + // Report it and exit, so a supervisor can restart the whole container. + // This is a process-exit event, not a heuristic probe, so there is no + // false-positive to tune. Death during startup keeps the old path: the + // startup code below reads `startupFailure` and renders a better message. + const engineRanFor = Date.now() - spawnedAt; + if (engineRanFor > ENGINE_STARTUP_GRACE_MS) { + console.error( + `[agentmemory] engine exited after ${Math.round(engineRanFor / 1000)}s ` + + `(code=${code} signal=${signal}); nothing can be served without it, exiting`, + ); + if (stderr.trim()) console.error(`[agentmemory] engine stderr:\n${stderr}`); + if (process.env["AGENTMEMORY_EXIT_ON_ENGINE_DEATH"] !== "0") { + process.exit(1); + } + } } }); child.unref(); return child; } +// Engine deaths inside this window are treated as startup failures, which the +// startup path reports with a better message. Later deaths mean a running +// deployment lost its engine, which is fatal to serving. +// +// Keep this SHORT. It is a hole in the only mechanism that catches engine death: +// an engine that dies inside the window after startup already completed leaves +// the container up and serving nothing, which is the original bug. A failed +// spawn surfaces within a second or two, so a few seconds is all the startup +// path needs. +const ENGINE_STARTUP_GRACE_MS = 5_000; + function startIiiBin(iiiBin: string, configPath: string): boolean { const s = p.spinner(); s.start(`Starting iii-engine: ${iiiBin}`); From 922b768d0479ef7b1721815b188b1d4caac22107 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 00:45:26 +0800 Subject: [PATCH 10/27] build(railway): build the image from this repo instead of the registry The Dockerfile installed the published @agentmemory/agentmemory package, so it never carried this repo's source. The deployed container ran 0.9.28 with none of the health fixes on this branch, and /opt/agentmemory/src did not exist. Any fork-level fix was invisible to production. Build the package here and install the packed tarball. Installing the tarball rather than copying files keeps npm placing the package at node_modules//, which reproduces the exact prefix entrypoint.sh hardcodes for the iii worker config. Moving that prefix would kill the container at boot under set -eu, so two build-time asserts now fail the build instead: one on the resolved iii-sdk version, one on the installed layout. No lockfile: .gitignore excludes it by repo policy, so it is absent from any git-based build context and npm ci cannot run. That trades build reproducibility away, which is the policy's cost rather than a choice made here. The builder upgrades npm first because node:22-slim ships 10.9.x, whose arborist fails this tree without a lockfile with "Cannot read properties of null (reading 'edgesOut')" -- reproduced on a clean git-only context. npm pack ships only the files allowlist, so the runtime install would otherwise resolve the repo's overrides unpinned. Derive them from the packed tarball so a new CVE pin cannot silently fail to reach the container. restartPolicyType becomes ALWAYS: the SIGTERM shutdown path ends in exit(0), which ON_FAILURE reads as success and would not restart. This also commits deploy settings that were already live but uncommitted: the in-memory OTEL exporter stays disabled (it drove the heap growth that crashed the container on 2026-08-23) and healthcheckTimeout stays at 60 for the BM25 startup backfill. The reason for the exporter divergence now lives next to the value rather than only in a test. The drift guard pins the four entrypoints against each other without line indices, so an insertion anywhere fails it rather than breaking it. --- .dockerignore | 10 ++++ deploy/README.md | 16 +++---- deploy/railway/Dockerfile | 63 ++++++++++++++++++++++---- deploy/railway/README.md | 39 +++++++++++++++- deploy/railway/entrypoint.sh | 4 +- deploy/railway/railway.json | 4 +- test/deploy-entrypoint-drift.test.ts | 68 ++++++++++++++++++++++++++++ 7 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 .dockerignore create mode 100644 test/deploy-entrypoint-drift.test.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..b96cb9a6c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +node_modules +.git +dist +eval +benchmark +test +.claude +.serena +docs +*.log diff --git a/deploy/README.md b/deploy/README.md index 91aa199e28..a84f653aaa 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,15 +1,10 @@ # One-click deploy templates Stand up agentmemory on managed infrastructure without rolling your own -Docker host. Each template ships a self-contained Dockerfile that pulls -`@agentmemory/agentmemory` from npm at build time and copies the iii -engine binary in from the official `iiidev/iii` image — no pre-built -agentmemory image required. Storage mounts at `/data`; an HMAC secret -is generated by the first-boot entrypoint and persisted to the volume. -The entrypoint overwrites the npm-bundled iii config with a -deploy-tuned one that binds `0.0.0.0` and uses absolute `/data` paths, -then drops privileges from `root` to `node` via `gosu` before -exec'ing the agentmemory CLI. +Docker host. Each template ships a self-contained Dockerfile. Fly, Render, and Coolify +install `@agentmemory/agentmemory` from npm at build time. Railway builds it +from this repo's source instead (see `deploy/railway/README.md`). All four +copy the iii engine binary in from the `iiidev/iii` image. | Platform | Pitch | Cost floor | |----------|-------|------------| @@ -95,6 +90,7 @@ agentmemory worker reg : 2.0 s healthcheck passes : ~9-10 s ``` -Every template's health-check `grace_period` (or compose +Railway's `healthcheckTimeout` is 60 s (the BM25 startup backfill needs it). +Every other template's health-check `grace_period` (or compose `start_period`) is set to 30 s for a 3x safety margin. Tune lower once you've measured your own platform's image-pull characteristics. diff --git a/deploy/railway/Dockerfile b/deploy/railway/Dockerfile index e094699881..eb884dc24c 100644 --- a/deploy/railway/Dockerfile +++ b/deploy/railway/Dockerfile @@ -2,9 +2,29 @@ ARG III_VERSION=0.11.2 FROM iiidev/iii:${III_VERSION} AS iii-image +# Build the package from this repo rather than pulling it from the registry, so +# fixes that live on a fork actually reach the deployed container. `npm pack` +# produces the same tarball shape the registry would serve, which keeps the +# install layout below identical to the previous registry install. +FROM node:22-slim AS builder +WORKDIR /build +# No lockfile: .gitignore:23 excludes it by repo policy, so it is absent from +# any git-based build context and `npm ci` cannot run. This trades build +# reproducibility away, which is the policy's cost, not a choice made here. +COPY package.json ./ +# node:22-slim ships npm 10.9.x, whose arborist fails this dependency tree with +# "Cannot read properties of null (reading 'edgesOut')" when resolving without a +# lockfile. Reproduced on a clean git-only context; npm 11 resolves it cleanly. +RUN npm install -g npm@11.19.0 --no-fund --no-audit \ + && npm install --no-fund --no-audit +COPY . . +# npm pack does not create --pack-destination, so make it first. +RUN mkdir -p /out \ + && npm run build \ + && npm pack --pack-destination /out + FROM node:22-slim -ARG AGENTMEMORY_VERSION=0.9.29 ARG III_VERSION=0.11.2 ARG III_SDK_VERSION=0.11.2 @@ -14,21 +34,44 @@ RUN apt-get update \ COPY --from=iii-image /app/iii /usr/local/bin/iii -# Install agentmemory into a dedicated prefix so the local package.json's -# `overrides` field pins iii-sdk down to match the engine (agentmemory's -# caret range `^0.11.2` otherwise resolves to 0.11.6, the version that -# requires the new sandbox-everything worker model the agentmemory CLI -# is not refactored for yet). `npm install -g` ignores overrides, hence -# the local prefix. +# Install into a dedicated prefix. The path below is load-bearing: entrypoint.sh +# writes the iii worker config to +# /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml +# under `set -eu`, so moving this prefix kills the container at boot rather than +# at build time. Installing the packed tarball keeps npm placing the package at +# node_modules//, which reproduces that exact path. WORKDIR /opt/agentmemory -RUN printf '{"name":"agentmemory-deploy","version":"1.0.0","private":true,"overrides":{"iii-sdk":"%s"}}\n' "${III_SDK_VERSION}" > package.json \ - && npm install "@agentmemory/agentmemory@${AGENTMEMORY_VERSION}" --omit=optional --no-fund --no-audit \ +COPY --from=builder /out/*.tgz /tmp/agentmemory.tgz +# The runtime install resolves the tarball's dependencies against the registry, +# so the repo's `overrides` never reach it on their own. These are CVE pins +# (added by 91c78e7, "clear all 8 audit vulns"). Read them out of the packed +# tarball rather than retyping them, so a new pin in package.json cannot ship +# unpinned here without anyone noticing. +RUN OVERRIDES="$(tar -xzOf /tmp/agentmemory.tgz package/package.json | node -p 'JSON.stringify((JSON.parse(require("fs").readFileSync(0,"utf8")).overrides)||{})')" \ + && printf '{"name":"agentmemory-deploy","version":"1.0.0","private":true,"overrides":%s}\n' "$OVERRIDES" > package.json \ + && npm install /tmp/agentmemory.tgz --omit=optional --no-fund --no-audit \ + && rm -f /tmp/agentmemory.tgz \ && ln -s /opt/agentmemory/node_modules/.bin/agentmemory /usr/local/bin/agentmemory +# The previous Dockerfile pinned iii-sdk through an `overrides` block to stop a +# caret range resolving past the engine. Both this repo and the published package +# now pin iii-sdk exactly, so the override is redundant. Assert the resolved +# version instead of carrying a workaround for a condition that may not hold: a +# mismatch fails the build here rather than at runtime in production. +RUN RESOLVED="$(node -p "require('/opt/agentmemory/node_modules/iii-sdk/package.json').version")" \ + && echo "resolved iii-sdk: ${RESOLVED} (expected ${III_SDK_VERSION})" \ + && [ "${RESOLVED}" = "${III_SDK_VERSION}" ] + +# Prove the build produced a usable install at the path the entrypoint expects. +# Without this the image can ship a broken layout that only fails at boot, where +# failure costs a restart-budget entry instead of a build. +RUN test -f /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml \ + && test -x /usr/local/bin/agentmemory + ENV AGENTMEMORY_III_VERSION=${III_VERSION} \ TINI_SUBREAPER=1 -COPY --chmod=0755 entrypoint.sh /usr/local/bin/agentmemory-entrypoint.sh +COPY --chmod=0755 deploy/railway/entrypoint.sh /usr/local/bin/agentmemory-entrypoint.sh EXPOSE 3111 diff --git a/deploy/railway/README.md b/deploy/railway/README.md index 9aad4fb32e..9ec128ee34 100644 --- a/deploy/railway/README.md +++ b/deploy/railway/README.md @@ -65,7 +65,9 @@ For an authenticated call, your client must send `Authorization: Bearer ## Viewer access (port 3113 stays internal) Railway only exposes the single public port from your service's -`PORT` env var (which we map to 3111). The viewer stays bound to +`PORT` env var. The container always serves on 3111 (the app reads +`III_REST_PORT`, never `PORT`), so the service's target port must be set to +3111 in the Railway dashboard. The viewer stays bound to localhost inside the container. `railway ssh` is an interactive shell only — it does not support `-L`-style port forwarding, so reach the viewer with one of the following. @@ -90,6 +92,39 @@ expose that port through a second Railway TCP Proxy, then use a native `ssh -L 3113:localhost:3113 -p ` from your laptop. This is the heavier path; option A is what most users will want. +## Self-healing + +Railway only queries `healthcheckPath` at deploy time and only restarts on a +process **exit**, so a container that stops serving without exiting is invisible +to the platform. Every wedge observed so far has the same shape: the iii engine +process dies and the node process keeps running, reconnecting forever. The +engine owns the REST listener, the stream port, and the state store, so nothing +can be served once it is gone. + +| Variable | Default | Effect | +|---|---|---| +| `AGENTMEMORY_EXIT_ON_ENGINE_DEATH` | **on** | Exits non-zero when the engine dies more than 5s after it was spawned, so the platform restarts the container. Set `0` to disable. | +| `AGENTMEMORY_HEALTH_ESCALATE` | unset (**off**) | Exits after 10 consecutive failures of the 30s KV probe. | + +`AGENTMEMORY_EXIT_ON_ENGINE_DEATH` is on by default because it acts on a +process-exit event rather than a probe: there is no threshold to tune and no +false positive to trade against. Deaths within the first 5 seconds keep the +startup-failure path, which reports a clearer message. + +`AGENTMEMORY_HEALTH_ESCALATE` is a probe and stays off. Do not enable it until an +external uptime check exists against `/agentmemory/livez`. It forces process +exits and `restartPolicyMaxRetries` is 10, so a wedge recurring on every boot +reaches a stopped deployment in under an hour with nothing notifying you. +Truthy spellings are `1`, `true`, `TRUE`. **`yes` and `on` are rejected.** + +`restartPolicyType` is `ALWAYS` rather than `ON_FAILURE`: the SIGTERM shutdown +path ends in `process.exit(0)`, which `ON_FAILURE` reads as success and would not +restart. + +An earlier revision of this file documented an in-container shell watchdog +(`AGENTMEMORY_WATCHDOG*`). That was removed on 2026-08-26 in favour of the +engine-exit handler above plus external uptime monitoring. + ## Rotate the HMAC secret ```bash @@ -132,5 +167,5 @@ See for the current rate card. or use the dashboard's manual snapshot feature. - The Dockerfile builds on Railway's builder on every deploy. First deploy is ~2 minutes; cached layers make subsequent rebuilds quick. - Pin `AGENTMEMORY_VERSION` / `III_VERSION` build args in the + Pin the `III_VERSION` build arg in the service's *Variables* tab to lock a specific release. diff --git a/deploy/railway/entrypoint.sh b/deploy/railway/entrypoint.sh index ffdd633395..7cde41a29f 100755 --- a/deploy/railway/entrypoint.sh +++ b/deploy/railway/entrypoint.sh @@ -67,7 +67,9 @@ workers: file_path: /data/stream_store - name: iii-observability config: - enabled: true + # false, unlike the other three deploy targets: the in-memory OTEL + # exporter drove heap growth that crashed the container (2026-08-23). + enabled: false service_name: agentmemory exporter: memory sampling_ratio: 1.0 diff --git a/deploy/railway/railway.json b/deploy/railway/railway.json index 43f52173b9..4272988850 100644 --- a/deploy/railway/railway.json +++ b/deploy/railway/railway.json @@ -7,8 +7,8 @@ "deploy": { "numReplicas": 1, "healthcheckPath": "/agentmemory/livez", - "healthcheckTimeout": 30, - "restartPolicyType": "ON_FAILURE", + "healthcheckTimeout": 60, + "restartPolicyType": "ALWAYS", "restartPolicyMaxRetries": 10, "requiredMountPath": "/data" } diff --git a/test/deploy-entrypoint-drift.test.ts b/test/deploy-entrypoint-drift.test.ts new file mode 100644 index 0000000000..0fb2215032 --- /dev/null +++ b/test/deploy-entrypoint-drift.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// The four deploy targets ship copies of one entrypoint, not a shared file: +// three of them build from their own directory and Docker cannot COPY from +// outside the build context. Copies drift silently -- the iii-observability flag +// sat at `true` in three files and `false` in a fourth with nothing recording +// that a decision had been made. +// +// This guard is deliberately not line-indexed. An earlier version pinned line +// numbers and broke on any insertion. +const read = (t: string) => + readFileSync( + fileURLToPath(new URL(`../deploy/${t}/entrypoint.sh`, import.meta.url)), + "utf8", + ); + +const TARGETS = ["railway", "fly", "render", "coolify"] as const; +const files = Object.fromEntries(TARGETS.map((t) => [t, read(t)])) as Record< + (typeof TARGETS)[number], + string +>; + +/** Body with comments and blank lines dropped, so a comment cannot mask drift. */ +const code = (s: string) => + s + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("#")) + .join("\n"); + +describe("deploy entrypoint drift", () => { + it("coolify and render remain byte-identical", () => { + expect(files.coolify).toBe(files.render); + }); + + // Railway is the only target that disables the in-memory OTEL exporter. The + // reason lives in deploy/railway/entrypoint.sh next to the value, so deleting + // this test cannot lose it. + it("only railway disables iii-observability", () => { + expect(files.railway).toMatch(/enabled: false/); + for (const t of ["fly", "render", "coolify"] as const) { + expect(files[t]).toMatch(/enabled: true/); + expect(files[t]).not.toMatch(/enabled: false/); + } + }); + + it("railway carries its reason next to the value", () => { + expect(files.railway).toMatch(/2026-08-23[\s\S]{0,120}enabled: false/); + }); + + // The real guard: with the observability line normalised away, railway's + // executable body must still equal render's. Any other divergence fails here, + // wherever it is inserted. + it("railway's executable body matches render apart from that one flag", () => { + const norm = (s: string) => code(s).replace(/enabled: (true|false)/, "enabled: X"); + expect(norm(files.railway)).toBe(norm(files.render)); + }); + + // Fly inserts its own block BEFORE the final exec rather than appending after + // it, so render's body up to that exec is the shared part. + it("fly shares render's body up to the final exec", () => { + const renderBody = code(files.render).split("\n").slice(0, -1); + const flyBody = code(files.fly).split("\n").slice(0, renderBody.length); + expect(flyBody).toEqual(renderBody); + }); +}); From f5b2ba6dc348d34c89a536ba0d8264eb695cd0da Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 00:45:44 +0800 Subject: [PATCH 11/27] docs: record the hang-resilience investigation and its corrections Keeps the reasoning that produced the two fixes, including the parts that turned out wrong, because the wrong turns are the reusable material. The first diagnosis blamed a stalled state worker inside a live process. It was an engine process death: the REST listener lives inside the engine, so it dies first rather than last, and the state::set timeout in the evidence is a consequence of the engine already being gone. That misreading came from one discarded assumption. The ECONNREFUSED reconnect stream was written off as background noise on the belief it starts at boot. It does not, and the arithmetic settles it: the SDK caps a reconnect attempt at 39 seconds, while one deployment logged attempt 807 over a span needing 70.9 seconds per attempt. Dividing elapsed time by the attempt number dates the outage to within minutes, and that signal sits in the log buffer already. Also corrected: a once-per-boot cap that bounds nothing at the deployment level because the exits are the subshell's, a SIGTERM grace justified as protecting a store it cannot reach, an acceptance example that contradicted the code, and a verification command that grepped a path the same document elsewhere says does not exist. --- ...-infra-agentmemory-hang-resilience-plan.md | 773 ++++++++++++++++++ 1 file changed, 773 insertions(+) create mode 100644 docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md diff --git a/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md new file mode 100644 index 0000000000..7e67c83735 --- /dev/null +++ b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md @@ -0,0 +1,773 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-plan-bootstrap +execution: code +created: 2026-08-25 +depth: deep +branch: fix/1223-heap-severity-denominator +--- + +# agentmemory hang resilience + +## Goal Capsule + +**Objective.** A wedged agentmemory backend recovers without a human, and a human +finds out that it happened. + +**Means.** Two independent detection layers plus a notification layer, and the +deploy-path change that lets the in-process layer actually ship (KTD1, KTD6). + +**Authority hierarchy.** Product Contract outranks Planning Contract. A user +instruction in session outranks both. Where this plan and +`~/.claude/rules/destructive-commands.md` disagree about running a deploy, the +rule wins: draft, show target, confirm. + +**Stop conditions.** Stop and ask before any `railway up`, before any `railway +redeploy`, and before opening the upstream PR. Stop if U1's source build produces +a `dist/` that fails U1's parity check. + +**Execution profile.** Verification-first. Every unit that changes runtime +behaviour carries a check that fails if the behaviour breaks. + +**Tail ownership.** `ce-work` implements. Multi-lens subagent review follows, per +the user's instruction and `~/.claude/rules/code-review-methodology.md`. + +--- + +## Product Contract + +### Summary + +On 2026-08-25 the agentmemory Railway service stopped serving while its process +stayed alive. Railway reported `Online` for about 8 hours. This was the second +occurrence. Nothing detected it and nothing recovered it. This plan makes the +failure self-healing and visible. + +### Problem Frame + +Two platform mechanisms should have caught it. Neither can, and the gap between +them is exactly where this failure lives. + +- `healthcheckPath: /agentmemory/livez` gates a **new deploy** only. Railway's + docs state: "Railway does not monitor the healthcheck endpoint after the + deployment has gone live." +- `restartPolicyType: ON_FAILURE` fires when the process **exits**. This process + hangs and never exits. + +**CORRECTED 2026-08-26: the root cause below is wrong, and the correction is +load-bearing for every unit in this plan.** This is not an application hang. The +**iii engine process dies and the node process keeps running.** Verified by +`railway ssh` into a wedged container: `/proc` held only `tini` and +`node /usr/local/bin/agentmemory`, with **no `iii` process at all**; +`/proc/net/tcp` had one listener, `127.0.0.1:3113` (the viewer, owned by node), +while 3111, 3112 and 49134 were all unbound; `/data/state_store.db` mtime equalled +`last_ok` exactly. + +The 3111 REST listener lives **inside the engine**, so it dies first, with the +engine, not last. That inverts F1 and F2 below. The `state::set` timeout in the +table is a **consequence** of the engine already being gone: an invocation with +no peer to answer it, timing out at the engine's own 180000 ms setting. The +engine died at 05:59:11Z, where the `ws` Sender errors are. + +A dead engine is therefore invisible to the platform. Worse, the application +already collects the signal that would identify it and then discards it: +`src/health/monitor.ts` probes `kv.set`/`kv.get` every 30 seconds and measures +event-loop lag, writes the snapshot to KV, and returns. `src/health/thresholds.ts` +never reads `kvConnectivity`, and nothing acts on `status: "critical"`. + +The observed failure sequence, from the Railway logs: + +| Time (UTC) | Event | +|---|---| +| 08-24 20:45:00 | Deploy `a201e22d` succeeds | +| 08-24 20:45:46 | `[iii] Reconnecting` attempt 1 (starts at boot, unrelated to the fault) | +| 08-25 05:59:11 | `ws` Sender errors in `node_modules/ws/lib/sender.js` | +| 08-25 06:05:10 | `error Compression failed: "Invocation timeout after 180000ms: state::set"` | +| 08-25 06:08:19 | Last application log line, then silence | + +Memory sat at 663.9 MB against an 8192 MB limit with CPU idle, so this was a hang +and not an out-of-memory event. + +### Requirements + +- **R1.** A process that stops serving on 3111 must be restarted without human + action. +- **R2.** The restart mechanism must not exhaust `restartPolicyMaxRetries: 10` + and leave the deployment permanently `CRASHED`. That outcome is strictly worse + than the bug it replaces. +- **R3.** An outage, and a restart loop, must reach a human without a human + polling for it. +- **R4.** Source-level fixes held in this fork must reach production. Today they + cannot. +- **R5.** The health monitor must act on the failure signal it already collects, + rather than recording it and returning. +- **R6.** No detection path may depend on `/agentmemory/health` while upstream + issue #1223 is unfixed in production, because that route returns 503 on a + healthy process. +- **R7.** The source-level fixes must be offered upstream to + `rohitg00/agentmemory`, not held only in the fork. + +### Actors + +- **A1. The container process.** `agentmemory` under `tini`, PID equal to the + entrypoint shell's PID after `exec`. +- **A2. The Railway platform.** Owns deploy-time healthchecks and the + `ON_FAILURE` restart policy. Sees process exits, not hangs. +- **A3. The operator.** Currently the only detector. This plan removes that role + from the recovery path and leaves it in the notification path. +- **A4. MCP clients.** Claude Code, Codex, and OpenCode. All read + `AGENTMEMORY_URL=http://127.0.0.1:8899`, so they degrade and recover together. + +### Key Flows + +- **F1. Listener death.** The 3111 listener disappears while the process lives. + The out-of-process watchdog detects it and forces an exit. Railway restarts. +- **F2. KV stall with a live loop.** `state::set` begins timing out while the + event loop still runs. The in-process monitor detects it and exits before the + listener dies. This is the path the real outage took at 06:05Z. +- **F3. Restart loop.** A deterministic wedge recurs on every boot. The + once-per-boot cap bounds it, and the notification layer surfaces it. + +### Acceptance Examples + +- **AE1.** Given a live container, when the 3111 listener stops answering, then + the container exits and restarts within roughly 4 minutes, and + `/agentmemory/livez` returns 200 without operator action. Covers R1, F1. +- **AE2.** Given a live node process, when the KV probe fails on **10** + consecutive 30-second collections, then the process exits. The threshold is 10, + not 3: `src/index.ts` documents `state::set` exceeding the SDK's 30 s timeout + under sustained hook load, and a 5 s probe cannot tell a slow store from a dead + one. An earlier draft said "before the HTTP listener dies" -- **that is wrong**, + the listener belongs to the engine and is already gone. Covers R5, F2. +- **AE3.** Given a wedge that recurs on every boot, when the watchdog acts, then + it acts at most once per container lifetime, and watchdog action alone never + drives the deployment to `CRASHED`. Covers R2, F3. +- **AE4.** Given a source-built image, when it is deployed, then + `grep -rl heapSizeLimit /dist/` returns at least one file. + **`dist/` is bundled, not per-module** — `tsdown` emits hash-suffixed chunks + (`src-CzgoepGU.mjs`, `index.mjs`) and there is no `dist/health/` directory, so + the assertion must be a recursive grep over `dist/` and never a fixed module + path. Verified against the live container: `heapSizeLimit` appears **nowhere** + in the deployed `dist/`, which is the current failing state this proves out of. + Covers R4. +- **AE5.** Given the backend is down, when the uptime check next runs, then a + notification reaches the operator without the operator looking. Covers R3. + +### Success Criteria + +- No operator action is required to recover from a hang. +- No hang exceeds roughly 5 minutes of downtime. +- Every recovery event and every restart loop produces a notification. +- The four health commits on this branch run in production. + +### Scope Boundaries + +**In scope.** The Railway deploy target. The entrypoint watchdog. The in-process +health escalation. The Dockerfile source build. External uptime monitoring. An +upstream PR. + +**Out of scope.** +- Fixing the `iii` engine's `state::set` hang at its source. The cause sits in a + third-party engine, the payoff is uncertain, and every layer here makes the + hang survivable regardless. +- Restructuring the health module's architecture. +- `deploy/fly/`, `deploy/render/`, and `deploy/coolify/`. Each carries the same + gap in its own `entrypoint.sh`. None is deployed, so none can be verified, and + unverified copies drift. Recorded as follow-up per the user's decision. +- The MCP 7-tool degradation. It is a symptom of backend downtime, not a separate + defect. It resolves when the backend stays up, and it needs an MCP server + restart per host because clients fetch `tools/list` once at connect. + +### Dependencies + +- Railway CLI, authenticated, linked at `~/rw-agentmemory`. +- `railway ssh` access for in-container verification. +- An uptime-check provider for U5. +- A GitHub account able to open a PR against `rohitg00/agentmemory`. The fork + `inix-x/agentmemory` already exists as `origin`. + +### Outstanding Questions + +- **Q1 (deferred, with a resolution path).** The running image is 0.9.28 while + the Dockerfile ARG pins 0.9.29, so a rebuild on the current Dockerfile would + bump the app version alongside any entrypoint change. The user parked this + decision. **U1 dissolves it:** once the image builds from the repo, the app + version stops being an npm pin and becomes the repo state, so there is no + separate bump to decide. If U1 is dropped or deferred, Q1 becomes blocking + again and must return to the user. Do not decide it silently. + +### Sources + +- Railway docs, healthchecks: "Railway does not monitor the healthcheck endpoint + after the deployment has gone live." Retrieved via context7, 2026-08-25. +- Upstream issue #1223, heap severity denominator. Fixed locally on this branch, + absent from the deployed package. +- Live container inspection via `railway ssh`, 2026-08-25. + +--- + +## Planning Contract + +### Key Technical Decisions + +- **KTD1. RETIRED 2026-08-26. The out-of-process shell watchdog was removed.** + The original reasoning was that two layers cover each other's blind spots. Two + things retired it. + + First, the root cause turned out to be **engine process death**, not an + application hang, and `src/cli.ts` already registers `child.on("exit")` on the + detached engine carrying the exit code, the signal, and its dying stderr. That + is a direct event, detected in milliseconds, with no probe, no polling, and no + false-positive threshold. It replaces ~93 lines of shell with a handler that + already existed. + + Second, U5 external monitoring is mandatory before either killer may be enabled + (KTD5), so it exists either way. `livez` is unauthenticated and static, so an + external check sees exactly what an in-container `curl` sees, **and** survives a + whole-container wedge the in-container subshell cannot. Railway exposes + `deploymentRestart` on its public API. + + What is lost, stated rather than glossed: an external monitor needs a Railway + token held at the provider, `deploymentRestart` takes a per-deploy id so it is a + small script rather than a bare URL, and an egress failure between monitor and + Railway can cause a restart the in-container watchdog would not have. + Governs R1, R5. + +- **KTD2. Probe `/agentmemory/livez`, never `/agentmemory/health`.** `livez` is + unauthenticated (`src/triggers/api.ts:180-192` registers `api::liveness` with no + `checkAuth`, verified by plain curl returning 200) and returns a static payload. + `health` returns 503 on a healthy process in production because #1223 is + unfixed there. Governs R6. + +- **KTD3. Hardcode 3111 in the watchdog. Never use `$PORT`.** The application + never reads `PORT`; `entrypoint.sh` writes `port: 3111` into the iii-http worker + config directly. `PORT` happens to equal 3111 today, verified in-container. If + Railway ever injected a different value, `${PORT:-3111}` would poll a dead port, + never arm, and fail silently. The literal is the more correct choice here. + +- **KTD4. SIGTERM first, and know that the flush it buys never happens.** An + earlier draft justified the grace period as protecting `/data/state_store.db` + from corruption. **Two separate findings retire that reasoning.** First, the + engine is a detached process in its own session that owns the store, so a + single-PID SIGTERM cannot reach it and it dies by namespace teardown either + way. Second, `shutdown` awaits `indexPersistence.save()` -> `kv.set` -> + `state::set`, which parks for the engine's 180 s invocation timeout against a + dead engine, while the grace is 15 s. The hard exit always wins, so the flush + never completes in the failure this targets. + + The grace stays for the case where the engine is alive and only node is being + restarted, where the flush does complete. It is not a corruption guard, and + the plan should not claim it is. Separately, Railway already sends SIGKILL + after 0 s of draining on every deploy, so the store has survived mid-write + kills routinely. Governs R2. + +- **KTD5. Ship the notification layer first. The once-per-boot cap is weaker than + it looks.** An earlier draft claimed the cap "bounds a deterministic wedge to + one restart per deploy." **That is wrong.** The watchdog's `exit 0` / `exit 1` + are the *subshell's* status, not the container's, and by the time either runs + PID 1 has already been signalled. The cap bounds one kill per **container**, + and containers are unbounded up to `restartPolicyMaxRetries`. Every restart is + a fresh container with a fresh watchdog. + + Measured cycle times, not assumed: the shell watchdog loops in roughly 3.5 to 4 + minutes, reaching `CRASHED` in about 35 to 40 minutes; the in-process layer + loops in about 5.3 minutes, reaching it in roughly 53 minutes. The `armed` + guard and the `until curl` loop only close the never-healthy-at-boot case, not + wedge-after-boot. + + So U5 is not a nicety that makes enabling tidier. It is the only thing that + makes a restart loop visible before the budget is gone, which is why it gates + U6. Governs R2, R3. + +- **KTD6. Build from source, and accept that it is not reproducible.** An earlier + draft of this decision claimed the repo carries a `package-lock.json` so + `npm ci` would be reproducible. **That was wrong.** The lockfile exists only in + a working tree: `.gitignore:23` excludes it under the comment "Lock files — + never commit". It is absent from `git archive HEAD`, so `COPY package-lock.json` + fails every git-based build. Verified by building from a git-only context. + + The build therefore runs `npm install` with no lockfile, which floats every + caret dependency. That is the cost of the repo's lockfile policy, not a choice + made here, and it applies equally to the registry install running today. State + it plainly rather than claiming a reproducibility this build does not have. + + Two further constraints found by building it rather than reasoning about it: + `node:22-slim` ships npm 10.9.x, whose arborist fails this tree without a + lockfile (`Cannot read properties of null (reading 'edgesOut')`), so the builder + upgrades to npm 11 first; and `npm pack` ships only the `files:` allowlist, so + the runtime install resolves dependencies against the registry and the repo's + `overrides` must be carried into the runtime manifest explicitly or the CVE pins + from `91c78e7` never reach the container. Governs R4. + +- **KTD7. Evaluate and count before the persist, never after.** In + `collectHealth`, the KV probe is raced against a 5-second timeout, but the + subsequent `await kv.set(KV.health, "latest", snapshot)` at `monitor.ts:94` is + **not raced**. During the real outage `state::set` was timing out at 180000 ms. + An escalation counter placed after that `await` would never increment during the + exact failure it targets, because the function would be parked on the persist + while the 30-second interval spawned more hung collections. The counter lives in + memory and is incremented before the persist. Governs R5. + +- **KTD8. The `overrides` block in the Dockerfile is vestigial.** Its comment + describes agentmemory resolving `iii-sdk` through a caret range. Both the + published 0.9.28 and the repo's 0.9.29 pin `iii-sdk` at exactly `0.11.2`, + verified in-container and in `package.json`. U1 should drop the workaround, and + must verify the resolved version after the build rather than assume it. + +### Assumptions + +- ~~The `[iii] Reconnecting` stream is background noise.~~ **RETRACTED + 2026-08-26. This was arithmetically false, and believing it produced the wrong + root cause above.** Port 49134 is the *engine's* port, so `ECONNREFUSED` there + means the engine is gone. iii-sdk caps one reconnect attempt at + `maxDelayMs 30000 x jitter 1.3` = **39 s**. Deployment `a201e22d` booted + 08-24T20:45:00Z and logged attempt **807** at 08-25T12:39:06Z: 57,200 s / 807 = + **70.9 s per attempt**, above the ceiling, so the stream cannot have begun at + boot. At the measured 30.6 s/attempt the onset is ~05:48Z, within 11 minutes of + the `ws` Sender errors. + + **The reconnect counter dates the outage.** Divide elapsed seconds by the + attempt number and compare against the 39 s ceiling. It is the earliest and + cheapest signal in the log buffer. +- `tini` exits non-zero when its child dies by signal, so Railway classifies the + exit as a failure and `ON_FAILURE` restarts. **U2 must verify this**, because + a clean exit 0 would not trigger a restart and the watchdog would be inert. +- A hang can leave the event loop running. Grounded: the application logged a + completed smart search at 06:08:19Z, three minutes after the `state::set` + timeout. This is what makes the in-process layer worth having. + +### Implementation Constraints + +- Stay on `fix/1223-heap-severity-denominator`. No branch switch, no worktree, + per `~/.claude/rules/no-branch-switching.md`. +- `deploy/railway/entrypoint.sh` and `deploy/railway/railway.json` are currently + modified and uncommitted, and those edits are what production runs. U0 commits + that state before anything stacks on it. +- The entrypoint is POSIX `sh`, not bash. `node:22-slim` provides `dash`. +- Never print `AGENTMEMORY_SECRET` or any `/data/.hmac` content. Handle by path + and by metadata, per `~/.claude/rules/triggr-credential-handling.md`. + +### Sequencing + +U0 gates everything. U5 gates the enablement of U2 and U4 per KTD5. U1 gates U3 +and U4 reaching production. U8 is independent and may run any time after U3/U4. + +**U1 and U2 both edit `deploy/railway/entrypoint.sh`, so they must be +serialized.** Run U1 first: it may relocate the install root, and U2's watchdog is +inserted relative to the final `exec` line that U1 may rewrite. Running them as +parallel branches means whichever is written second clobbers the first. + +``` +U0 (commit + backup) + | + +--> U1 (source build, edits entrypoint.sh) + | + +--> U2 (watchdog, edits entrypoint.sh, shipped DISABLED) --+ + | | + +--> U3 --> U4 -------------------------------------------- + + | | + | U5 --> U6 (enable) --> U7 (verify) + | + +--> U8 (upstream PR) +``` + +--- + +## Implementation Units + +### U0. Commit the deployed state and back up the volume + +**Goal.** Establish a known-good rollback point before anything changes. + +**Requirements.** R2. + +**Files.** `deploy/railway/entrypoint.sh`, `deploy/railway/railway.json`. + +**Approach.** Both files are modified and uncommitted, and both are what +production runs. Commit them as-is on the current branch so later changes are +separable. Then back up `/data` using the command in `deploy/railway/README.md`. +Record the current deployment id and image digest so the rollback target is +written down rather than remembered. + +**Test Scenarios.** +- `git status --short -- deploy/railway/` reports clean after the commit. +- The backup archive exists locally and is non-empty. +- The recorded digest matches `railway deployment list --json` for the live + deployment. + +**Verification.** `git log -1 --stat -- deploy/railway/` shows both files. The +archive's size is within an order of magnitude of the volume's 0.8 GB. + +--- + +### U1. Build the image from repo source + +**Goal.** Make fork-level source changes reach production. Today they cannot. + +**Requirements.** R4. Dissolves Q1. + +**Files.** `deploy/railway/Dockerfile`, `deploy/railway/entrypoint.sh`, +`.dockerignore` (new). Plus one **Railway service setting**, which is not a file. + +**`rootDirectory` is NOT in `railway.json`.** Verified: the file's keys are +`$schema`, `build{builder,dockerfilePath}`, `deploy{...}`, and neither +`serviceManifest` nor `fileServiceManifest` carries `rootDirectory`, yet the +deployment meta reports `rootDirectory: deploy/railway`. It is a dashboard/API +service setting. **Editing `railway.json` will not change the build context.** If +this unit only edits files, the context stays `deploy/railway/`, `src/` never +uploads, and U1 silently no-ops while appearing to succeed. Change the service +setting to the repo root and repoint `dockerfilePath` accordingly. + +**Approach.** The current Dockerfile has two `COPY` lines, neither touching +`src/`, and no build step. It installs the published package from npm, which is +why `/opt/agentmemory/src` does not exist in the image and the deployed version +is 0.9.28. Change the build context to the repo root, copy the manifest and +lockfile, run `npm ci`, copy the source, run `npm run build` (`tsdown`), and point +the `agentmemory` bin at the built `dist/cli.mjs`. Prefer a multi-stage build so +dev dependencies do not ship. Add `.dockerignore` to keep `node_modules`, +`.git`, `dist`, and `eval/` out of the upload. Drop the vestigial `overrides` +block per KTD8, then verify the resolved `iii-sdk` rather than assuming. + +Note that `dist/` is gitignored, so the image must build it and must never expect +a prebuilt copy. + +**The entrypoint is hard-coupled to the npm install layout, and this is the way +U1 fails at boot.** `entrypoint.sh` runs under `set -eu` and does: + +```sh +III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" +cat > "$III_CONFIG" <<'EOF' +``` + +If U1 moves the install root, that `cat >` writes into a directory that no longer +exists, `set -e` aborts, and the container dies **before the app starts**. Railway +reads that as a failure and retries, so a botched U1 burns all ten retries on the +first deploy and lands in `CRASHED` — precisely the R2 outcome this plan exists to +prevent. `npm run build` is `tsdown && cp iii-config.yaml dist/`, so the config +always lands at `/dist/iii-config.yaml`. + +Choose one deliberately and write down which: **either** preserve the exact +`/opt/agentmemory/node_modules/@agentmemory/agentmemory/` prefix, **or** update +`III_CONFIG` to the new root. The same applies to the final +`exec gosu "$RUN_AS" agentmemory "$@"`, which resolves `agentmemory` on PATH +through the Dockerfile's symlink into `node_modules/.bin/`. U1 must recreate an +equivalent or that line fails too. + +**Test Scenarios.** +- The image builds from a clean context. +- `node -e "require('/opt/agentmemory/.../package.json').version"` inside the + built image reports 0.9.29, matching the repo. +- `grep -rl heapSizeLimit /dist/` inside the image returns at least + one file. This is AE4. Use a recursive grep: `dist/` is bundled and + `dist/health/thresholds.js` does not exist. +- **The container boots at all.** `III_CONFIG` resolves and `agentmemory` is on + PATH. Test this before any deploy, because failure here is a boot loop that + eats the retry budget. +- The resolved `iii-sdk` is 0.11.2. +- `/agentmemory/livez` returns 200 from the built image. + +**Verification.** Build locally first. Only then consider a deploy, which is +gated by U7 and by the destructive-commands confirmation. + +--- + +### U2. REMOVED — out-of-process watchdog + +**Deleted 2026-08-26**, superseded by the engine-exit handler (U9) and U5. See +KTD1 for the reasoning. Removed: the ~93-line watchdog block from +`deploy/railway/entrypoint.sh`, four `AGENTMEMORY_WATCHDOG*` variables, their +validation guards, and the tests that exercised them. `curl` **stays** in the +image: `deploy/railway/README.md` documents `railway ssh` plus +`curl http://localhost:3113` as the in-container viewer check, and that path was +used for the 2026-08-26 forensics that corrected the root cause. + +### U9. Exit when the engine dies + +**Goal.** Turn the actual failure into a process exit, at the place that already +detects it. + +**Requirements.** R1, R2. Replaces F1. + +**Files.** `src/cli.ts`. + +**Approach.** The engine is spawned detached and already has a `child.on("exit")` +handler that captures the exit code, the signal, and up to 16KB of stderr, then +logs under `vlog` and returns. Past a 60-second startup grace, report the death on +`console.error` with that stderr and exit non-zero. Deaths inside the grace keep +the existing path, which renders a better startup message. + +Default **on**, unlike the probe-based layers: this acts on a process-exit event, +so there is no threshold to tune and no false positive to trade against. The +engine owns the REST listener, the stream port, and the state store, so nothing +can be served once it is gone. Opt out with `AGENTMEMORY_EXIT_ON_ENGINE_DEATH=0`. + +**Test Scenarios.** Death inside the grace keeps the startup path. Death after it +exits non-zero. The opt-out suppresses the exit. The stderr reaches the log. + +**Verification.** `npm test`, plus a container run that kills the engine and +observes the exit code. + +### U2-original (superseded, kept for the record) + +**Goal.** Convert a hang into an exit so the existing restart policy can act. + +**Requirements.** R1, R2. Implements F1. + +**Files.** `deploy/railway/entrypoint.sh`. + +**Approach.** Insert a backgrounded POSIX `sh` loop immediately before the final +`exec gosu`. Capture `$$` first: `exec` preserves the PID, so the shell's PID is +the PID the application will hold. The loop waits for a first successful `livez` +before arming, so a slow BM25 startup backfill can never trigger it, and a boot +that never succeeds is left to Railway's deploy-time healthcheck. After arming, it +polls every 60 seconds; on `WATCHDOG_FAILS` consecutive failures it sends SIGTERM, +waits `WATCHDOG_GRACE` seconds, then sends SIGKILL only if the process survives. +It then exits, which enforces the once-per-boot cap in KTD5 structurally rather +than by a flag. + +**Ship it disabled.** Default `AGENTMEMORY_WATCHDOG=0` in this unit. U6 flips it +after U5 exists. + +The loop's logic was drafted and exercised during planning, and all five +behaviours passed: never arms against a server that was never up; does not fire +at 2 of 3; fires at exactly 3 of 3; escalates SIGTERM to SIGKILL when SIGTERM is +ignored; a short blip resets the counter without a kill. `ce-work` should re-run +these rather than trust the record. + +**Test Scenarios.** +- Never arms when the endpoint has never returned 200. +- Does not fire at `WATCHDOG_FAILS - 1`. +- Fires at exactly `WATCHDOG_FAILS`. +- Escalates to SIGKILL when the target ignores SIGTERM. +- A recovery shorter than the threshold resets the counter and no kill occurs. +- **`AGENTMEMORY_WATCHDOG=0` produces no watchdog process at all.** +- **`tini` exits non-zero when its child is signalled**, so `ON_FAILURE` fires. + This is the Planning Contract assumption that must be proven, not assumed. A + clean exit 0 would make the whole unit inert. + +**Verification.** `sh -n` for syntax, then the behavioural harness against a +throwaway HTTP server. Confirm the exit code with a container-local run. + +--- + +### U3. Evaluate `kvConnectivity` in the health thresholds + +**Goal.** Stop discarding the signal that identifies this exact failure. + +**Requirements.** R5. Implements part of F2. + +**Files.** `src/health/thresholds.ts`, `test/health-thresholds.test.ts` +(**existing, append only**). + +**`test/health-thresholds.test.ts` ALREADY EXISTS** — 174 lines, 9 tests, +carrying the regression coverage for issue #158 and for the #1223 denominator +work in commit `44e5372`. **Append to it. Never overwrite it.** It already +defines a `snap()` fixture helper whose default includes `kvConnectivity`, so +reuse that helper rather than introducing a second fixture. + +An earlier draft of this plan claimed no health test file existed. That claim +came from a `find` whose output was swallowed by the `rtk` wrapper, and acting on +it destroyed the file (recovered from `HEAD`). Read a negative search result +through `rtk proxy ` before trusting it. + +**Approach.** `collectHealth` populates `snapshot.kvConnectivity` with `status`, +`latencyMs`, and an optional `error`, using a 5-second race. `evaluateHealth` +checks `connectionState`, `eventLoopLagMs`, `cpu`, and `memory`, and never reads +`kvConnectivity`. Add it: `status === "error"` is critical, with an alert string +matching the existing `snake_case_value` convention used by +`connection_disconnected` and `event_loop_lag_critical_NNNms`. Follow the existing +alert vocabulary rather than inventing a new shape. + +**Test Scenarios.** +- A snapshot with `kvConnectivity.status === "error"` evaluates to `critical` and + carries a `kv_*` alert. +- A snapshot with `status === "ok"` does not add an alert. +- An absent or malformed `kvConnectivity` does not throw and does not evaluate to + critical. The field is optional in older persisted snapshots, so this is a real + compatibility path, not a hypothetical. +- The existing heap, CPU, event-loop, and connection assertions still pass. + +**Verification.** `npm test`, scoped to the health tests. + +--- + +### U4. Escalate sustained critical health to a process exit + +**Goal.** Let the in-process layer act on what it detects, catching a wedge while +the event loop still runs. + +**Requirements.** R1, R2, R5. Implements F2. + +**Files.** `src/health/monitor.ts`, `test/health-monitor.test.ts` (new file). +`test/health-thresholds.test.ts` **already exists** — see U3. An earlier draft +claimed no health test file existed; that claim was wrong and acting on it +destroyed the file once already. + +**Approach.** `registerHealthMonitor` currently ends `collectHealth` with a +persist and a return. Add an in-memory consecutive-critical counter. **Increment +and evaluate it before the `await kv.set(KV.health, "latest", snapshot)` call at +`monitor.ts:94`, never after** — that persist is not raced against any timeout, +and during the real outage `state::set` hung for 180 seconds, so a counter behind +it would never advance during the failure it exists to catch (KTD7). On reaching +the threshold, log a clear reason and exit so `ON_FAILURE` restarts. + +Reuse the existing SIGTERM `shutdown` path at `src/index.ts:629-630` rather than +calling `process.exit` directly, so the KV store flushes. Fall back to a hard +exit only if shutdown does not complete within a grace window. + +Gate the whole behaviour behind an environment flag, default **off**, matching +U2's disabled-by-default posture and KTD5. + +**Test Scenarios.** +- N consecutive critical snapshots trigger exactly one escalation. +- A single healthy snapshot between criticals resets the counter. +- With the flag off, no escalation occurs regardless of snapshot status. +- The counter advances when the persist is slow or rejects. This is the + regression test for KTD7 and the one most likely to be got wrong. +- Escalation runs the graceful shutdown path, not a bare `process.exit`. + +**Verification.** `npm test`. Assert on the escalation decision, not on a real +process exit. + +--- + +### U5. External uptime monitoring + +**Goal.** Make an outage and a restart loop reach a human without polling. + +**Requirements.** R3. Prerequisite for U6 per KTD5. + +**Files.** None in this repo. Configuration lives with the provider. Record the +setup in `deploy/railway/README.md`. + +**Approach.** Point an uptime check at +`https://agentmemory-production-0c12.up.railway.app/agentmemory/livez` on roughly +a 5-minute interval, alerting to a channel the user actually reads. The route +needs no auth, verified. This is the only layer that catches a restart loop +burning the retry budget, which is precisely the failure mode the other two +layers can create. + +**Test Scenarios.** +- The check reports healthy against the live service. +- A deliberate outage window produces an alert. +- The alert reaches the intended channel, not only the provider's dashboard. + +**Verification.** Confirm one real alert end to end. An untested alert path is +not a notification layer. + +--- + +### U6. Enable the watchdog and the escalation + +**Goal.** Turn on the killers, once notification exists. + +**Requirements.** R2, R3. + +**Files.** `deploy/railway/entrypoint.sh` default, or a Railway service variable. + +**Approach.** Flip `AGENTMEMORY_WATCHDOG` and U4's flag to on. Prefer Railway +service variables over a Dockerfile default so a rollback needs no rebuild. Do +not perform this unit until U5 has produced one verified alert. + +**Test Scenarios.** +- Both layers report armed in the deploy logs. +- A synthetic wedge in a throwaway environment produces exactly one restart. + +**Verification.** Read the deploy logs for both arming lines. + +--- + +### U7. Deploy and verify end to end + +**Goal.** Prove the whole chain on the live service. + +**Requirements.** All. + +**Files.** None. + +**Approach.** `railway up` from the repo root. **Confirm with the user first**, +per `~/.claude/rules/destructive-commands.md`: show the target project, +environment, service, and the rollback digest from U0. Watch the deployment to a +terminal state; never call it good on `railway status` alone, which reported +`Online` throughout the outage. + +**Test Scenarios.** +- AE4: `heapSizeLimit` is present in the deployed `dist/health/thresholds.js`. +- `/agentmemory/livez` returns 200 externally. +- The viewer at `http://127.0.0.1:8899/agentmemory/viewer` returns 200. +- `_proxy/status` shows a small `seconds_since_last_ok`. +- `memory_recall` through MCP returns real observations. +- Both watchdog layers log that they armed. + +**Verification.** Run every check above. Per +`~/.claude/rules/verification-no-dismiss.md`, trace any non-green item to ground +before declaring the deploy verified. + +--- + +### U8. Upstream PR to `rohitg00/agentmemory` + +**Goal.** Offer the fixes upstream instead of holding them only in the fork. + +**Requirements.** R7. + +**Files.** The U3 and U4 changes, plus the four existing health commits on this +branch. + +**Approach.** `origin` is `inix-x/agentmemory` and `upstream` is +`rohitg00/agentmemory`. Open a PR carrying the `kvConnectivity` evaluation and the +escalation. Reference issue #1223 for the heap work already committed here. +Confirm with the user before opening, per the destructive-commands rule: a PR is +outward-facing. + +**Test Scenarios.** Upstream CI passes. The PR description states the failure +mode, the evidence, and the mechanism. + +**Verification.** The PR URL is surfaced to the user. + +--- + +## Verification Contract + +Repo-specific commands: + +```bash +npm test # vitest, excludes test/integration.test.ts +npm run test:all # includes integration +npm run build # tsdown -> dist/ +sh -n deploy/railway/entrypoint.sh +``` + +Live-service checks: + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' \ + https://agentmemory-production-0c12.up.railway.app/agentmemory/livez +curl -sS http://127.0.0.1:8899/_proxy/status +# Recursive: dist/ is BUNDLED, so dist/health/thresholds.js does not exist. +cd ~/rw-agentmemory && railway ssh --service agentmemory -- \ + "grep -rl heapSizeLimit /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/ | wc -l" +``` + +Quality gates: no unit is done while its own tests fail. No deploy is verified +while any check above is non-green or untraced. + +--- + +## Definition of Done + +**Global.** +- A hang recovers with no operator action, within roughly 5 minutes. +- A restart loop produces a notification rather than silence. +- The deployed image contains this branch's health fixes, proven by AE4. +- The upstream PR is open. +- No dead-end or experimental code remains in the diff. A long autonomous run + accumulates abandoned attempts; declaring done requires removing them. + +**Per unit.** Every Test Scenario passes, and every unit that changes runtime +behaviour leaves behind a check that fails if that behaviour breaks. + +**Explicitly not done** if: `railway status` alone is used as proof, the +notification path is configured but never fired once, or Q1 is resolved silently +in the event U1 is dropped. From 6a27bca3437f970c38568438cd124690b956f8f7 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 12:52:47 +0800 Subject: [PATCH 12/27] fix(state): reclaim index generations the manifest can no longer name saveShardedIndex wrote each new generation's shards, published a manifest naming it, then deleted the previous generation only when its manifest read had succeeded. That read is the first thing the function does, and it fails under exactly the load that makes saves slow, so `previous` became null and the whole preceding generation was stranded. Nothing revisits it: a later save only inspects its own predecessor, and loadShardedData reads only the shards the current manifest lists. Two further paths reached the same state. The manifest catch block ends in a throw, so a write that threw after committing skipped the cleanup below it. And a kill between the first shard write and the cleanup left a generation nothing had recorded. Verified on a deployed store: 8 dead BM25 generations, 339 MB, 33.5% of a 1013 MB store, the oldest 13 days old, against 35 MB of actual observations. Every orphan's generation id decodes to a restart or an incident. Record each generation in a ledger beside its manifest, written before the first shard write, and reclaim from it after publish and again on load. StateKV lists keys within a scope and never scopes by prefix, so a generation that is not recorded cannot be found again by anything. Serialise saves. scheduleSave fires save() unawaited from a timer while flushIndexSave awaits save() on every delete path, and stop() clears the timer without awaiting a save already running, so saves overlap by construction. Two of them read-modify-writing one ledger drop each other's entries, and the one that publishes second reclaims the shards the first is still writing, leaving it to publish a manifest naming data that is already gone. The queue never coalesces: flushIndexSave awaits save() to make a delete durable, so returning an in-flight promise that started before the delete would report success for a snapshot without it. The load-time reclaim joins the same queue. An unreadable ledger skips that cycle's tracking and reclaim rather than failing the save. state::get timing out is the condition this bug appears under, and aborting there would stop persisting the index entirely, which costs a full-corpus rebuild. A ledger whose shape is not recognised is left untouched, never rewritten. Reclaim keeps malformed entries instead of iterating them, so a GC step can never fail a load, and emits one audit row per sweep rather than one per shard, per the policy in src/functions/audit.ts. Each index saves independently, since one try around both would let a BM25 failure stop the vector index persisting, and a lost vector index is never rebuilt: both rebuild triggers key on the BM25 size. A failed delete keeps its shards listed and retries, rather than stranding them. The ledger cannot discover generations orphaned before it existed; those need a one-time reclaim. Closes #1115 --- src/state/index-persistence.ts | 402 +++++++++++++++++++++++++++++++-- test/index-persistence.test.ts | 377 +++++++++++++++++++++++++++++++ 2 files changed, 754 insertions(+), 25 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..8b0d125423 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -24,6 +24,27 @@ type IndexShardManifest = { chars: number; }; +// Suffix for the reclaim ledger that sits beside each manifest. +const GC_LEDGER_SUFFIX = ":gc"; + +// Stands in for a manifest written before generations were recorded, whose +// `generation` field is absent. Cannot collide with createIndexGeneration(). +const PRE_LEDGER_GENERATION = "pre-ledger"; + +// Every generation whose shards may still be on disk, live one included. The +// manifest alone cannot answer that: it names only the generation that is +// current, so a generation stranded by a failed read, a throw after commit, or +// a kill mid-cleanup becomes unreachable the moment the next manifest replaces +// it. Nothing else enumerates shard scopes — StateKV lists keys within a scope, +// not scopes by prefix — so what is not recorded here can never be found again. +type IndexGcLedger = { + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; +}; + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; @@ -67,7 +88,8 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private queue: Promise = Promise.resolve(); constructor( private kv: StateKV, @@ -83,7 +105,7 @@ export class IndexPersistence { // under sustained iii-engine write timeouts (issue #204). Funnel // rejections through logFailure() instead. this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); + this.save().catch((err) => this.logFailure("index", err)); }, DEBOUNCE_MS); } @@ -92,13 +114,52 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } + return this.enqueue(() => this.runSave()); + } + + /** + * Serialise everything that read-modify-writes the gc ledger. + * + * Two saves genuinely overlap here: scheduleSave fires save() unawaited from + * a timer, flushIndexSave awaits save() on every delete path + * (src/functions/search.ts), and stop() clears the timer without awaiting a + * save already running. Overlapping saves drop each other's ledger entries, + * and worse, a save that publishes second reclaims the shards the first is + * still writing — leaving the first to publish a manifest naming data that + * is already gone, which fails the next load closed. + * + * Queue, never coalesce. flushIndexSave awaits this to make a delete + * durable, so handing back an in-flight promise that started before the + * delete would report success for a snapshot that does not contain it. The + * cost is real: a delete-path flush waits out the save ahead of it. + */ + private enqueue(work: () => Promise): Promise { + // runSave never rejects and the tail below always fulfils, so the queue + // cannot be left rejected and needs no rejection handler here. + const run = this.queue.then(work); + this.queue = run.then( + () => undefined, + () => undefined, + ); + return run; + } + + private async runSave(): Promise { + // Each index fails on its own. One try around both would let a BM25 + // failure stop the vector index persisting at all, and a lost vector index + // is never rebuilt: both rebuild triggers key on the BM25 size + // (src/index.ts, src/functions/search.ts). try { await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { + } catch (err) { + this.logFailure("BM25", err); + } + if (this.vector) { + try { await this.saveVectorIndex(this.vector.serialize()); + } catch (err) { + this.logFailure("vector", err); } - } catch (err) { - this.logFailure(err); } } @@ -129,16 +190,19 @@ export class IndexPersistence { } } - private logFailure(err: unknown): void { + private logFailure(index: string, err: unknown): void { const now = Date.now(); // Throttle: persistence failures under load arrive in bursts // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + // noise without information. Throttled PER INDEX, so a vector failure + // right after a BM25 one is not swallowed — the two fail independently + // now, and at 3am you need to know which one stopped persisting. + const lastAt = this.lastFailureLogAt.get(index) ?? 0; + if (now - lastAt < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(index, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save ${index} index`, { code, message, hint: @@ -192,6 +256,16 @@ export class IndexPersistence { chunks.push(chunk); } + // Record the generation BEFORE the first shard write. A kill anywhere from + // here to the manifest publish would otherwise leave shards on disk that + // nothing references and nothing can enumerate. + const tracked = await this.trackGeneration( + manifestKey, + generation, + shards, + previous, + ); + const writeResults = await Promise.allSettled( shards.map(async (shard, index) => { const chunk = chunks[index] ?? ""; @@ -211,7 +285,14 @@ export class IndexPersistence { (result): result is PromiseRejectedResult => result.status === "rejected", ); if (failedWrite) { - await this.deleteShards(shards, "shard_write_rollback"); + const allGone = await this.deleteShards(shards, "shard_write_rollback"); + // Drop the entry only if every shard actually went. A shard that survived + // its delete is unnameable once its entry is gone, which is the contract + // this file states for reclaim: a delete failure costs a retry, never a + // stranded generation. + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } throw failedWrite.reason; } @@ -249,23 +330,275 @@ export class IndexPersistence { error: errorMessage(err), }); } else { - await this.deleteShards(shards, "manifest_publish_rollback"); + const allGone = await this.deleteShards( + shards, + "manifest_publish_rollback", + ); + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } } throw err; } await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup"); - if (previous?.v === 1 && Array.isArray(previous.shards)) { - const currentShardIds = new Set( - shards.map((shard) => `${shard.scope}\0${shard.key}`), + // Only reclaim when this generation is actually in the ledger. Reclaiming + // against a ledger that does not list us would treat live shards as dead. + if (tracked) { + await this.reclaimGenerations(manifestKey, generation); + } else if (previous?.v === 1 && Array.isArray(previous.shards)) { + // The ledger was unusable this cycle, so nothing above will ever revisit + // `previous`. Fall back to the pre-ledger cleanup: the manifest just + // published supersedes it, and saves are serialised, so its shards are + // dead. Without this, a run of unusable cycles orphans one generation + // each — strictly worse than the code this replaced, which always had + // this path. + const liveIds = new Set( + shards.map((shard) => `${shard.scope}\u0000${shard.key}`), + ); + await this.deleteShards( + previous.shards.filter( + (shard) => + isValidShardDescriptor(shard) && + !liveIds.has(`${shard.scope}\u0000${shard.key}`), + ), + "previous_generation_cleanup", + ); + } + } + + /** Drop a generation's entry after its shards have been rolled back. */ + private async untrackGeneration( + manifestKey: string, + generation: string, + ): Promise { + try { + const ledger = await this.readLedger(manifestKey); + const generations = ledger.generations.filter( + (entry) => entry?.generation !== generation, ); - for (const shard of previous.shards) { - if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue; - await this.deleteShards([shard], "previous_generation_cleanup"); + if (generations.length === ledger.generations.length) return; + await this.kv.set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations, + }); + } catch { + // Best effort. A surviving entry costs a retry on the next reclaim, and + // this runs while a save is already failing — never make that worse. + } + } + + private gcKey(manifestKey: string): string { + return `${manifestKey}${GC_LEDGER_SUFFIX}`; + } + + private async readLedger(manifestKey: string): Promise { + // Throws rather than returning a blank ledger, because a blank one would + // be written straight back over whatever is stored. Callers catch it and + // skip tracking for that cycle; they must never treat it as "no ledger". + const stored = await this.kv.get( + KV.bm25Index, + this.gcKey(manifestKey), + ); + if (stored == null) return { v: 1, generations: [] }; + // Present but unrecognised: a newer version, or a rollback to this build + // after one that wrote a different shape. Overwriting drops every + // generation it tracked, so leave it exactly where it is. + if (stored.v !== 1 || !Array.isArray(stored.generations)) { + throw new Error( + `index gc ledger ${this.gcKey(manifestKey)} has an unrecognised shape ` + + `(v=${String((stored as { v?: unknown }).v)}); refusing to overwrite it`, + ); + } + return stored; + } + + private async trackGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + try { + // The WHOLE body runs under this guard, not just the ledger read. A + // malformed ledger entry or manifest shard would otherwise throw a + // TypeError out of saveShardedIndex before any shard write, leaving one + // throttled log line per 60s as the only trace while BM25 stopped + // persisting for good. isValidShardDescriptor exists in this file + // because a per-shard-malformed manifest is already its threat model. + return await this.recordGeneration( + manifestKey, + generation, + shards, + previous, + ); + } catch (err) { + // A state::get brownout is the exact condition this bug appears under, + // so this path is not rare. Aborting the save here would stop persisting + // the index at all, which is worse than the leak being fixed. Writing a + // fresh ledger would drop every generation the stored one lists. Do + // neither: let the shards and manifest land, skip this cycle's tracking + // and its reclaim, and leak at most one generation instead of one per + // brownout. + // + // That leak can OUTLIVE the unreadable window. This generation publishes + // untracked; the next save re-seeds it from `previous`, but that read is + // itself `.catch(() => null)`, so if it also fails the generation is in + // no ledger and reclaim only ever looks at what precedes the live entry. + // With no scope enumeration in StateKV, nothing can find it again. + // Throttled for the same reason every other failure log here is: an + // unusable ledger stays unusable, so this fires on every debounce. + const throttleKey = `gc:${manifestKey}`; + const now = Date.now(); + if (now - (this.lastFailureLogAt.get(throttleKey) ?? 0) >= FAILURE_LOG_THROTTLE_MS) { + this.lastFailureLogAt.set(throttleKey, now); + logger.warn( + "index persistence: gc ledger unavailable, skipping reclaim", + { manifestKey, message: errorMessage(err) }, + ); } + return false; } } + private async recordGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + const ledger = await this.readLedger(manifestKey); + // Optional-chained so one malformed entry costs that entry, not the whole + // cycle's tracking. reclaimGenerations tolerates them the same way. + const known = new Set(ledger.generations.map((entry) => entry?.generation)); + + // Seed the generation the pre-ledger code left live, so upgrading does not + // strand it. Only reachable while `previous` is readable, which is exactly + // the case the old cleanup already handled. + // + // `generation` is optional on the manifest and older stores really do omit + // it, so fall back to a sentinel rather than skipping the seed. The + // sentinel cannot collide with createIndexGeneration()'s `idx_` ids, and + // since every manifest written from here on carries a generation, the + // seeded entry always ends up preceding a live one and gets reclaimed. + if (previous?.v === 1 && Array.isArray(previous.shards)) { + const previousGeneration = previous.generation ?? PRE_LEDGER_GENERATION; + if (!known.has(previousGeneration)) { + ledger.generations.push({ + generation: previousGeneration, + shards: previous.shards + .filter(isValidShardDescriptor) + .map(({ scope, key }) => ({ scope, key })), + }); + known.add(previousGeneration); + } + } + + if (!known.has(generation)) { + ledger.generations.push({ + generation, + shards: shards.map(({ scope, key }) => ({ scope, key })), + }); + } + + // ponytail: unbounded while EVERY delete keeps failing — each debounce + // appends a generation carrying all its shard descriptors, and + // reclaimGenerations only rewrites when something was actually reclaimed. + // Add a length cap if that ever shows up in the wild; a cap is not one + // line, and this needs a total delete outage to reach. + await this.kv.set( + KV.bm25Index, + this.gcKey(manifestKey), + ledger, + ); + return true; + } + + /** + * Delete every tracked generation except the live one, then rewrite the + * ledger with whatever survived. A shard whose delete failed stays listed and + * is retried on the next save or load, so a delete failure costs a retry + * instead of stranding the generation for good. + */ + private async reclaimGenerations( + manifestKey: string, + liveGeneration: string | undefined, + ): Promise { + if (!liveGeneration) return; + const ledger = await this.readLedger(manifestKey).catch(() => null); + if (!ledger) return; + + // Reclaim strictly what precedes the live generation in the ledger, which + // trackGeneration appends to in creation order. Anything at or after the + // live entry is either live or a save still in flight: setIndexPersistence + // runs before load() in src/index.ts, so a request arriving during boot can + // have a save writing shards while this reclaim runs, and deleting those + // would publish a manifest whose shards are already half gone. A generation + // stranded after the live one is not lost, only deferred — it becomes + // reclaimable as soon as a newer generation is published. + const liveIndex = ledger.generations.findIndex( + (entry) => entry?.generation === liveGeneration, + ); + // Live generation untracked (a pre-ledger store, or a manifest written + // before this shipped). Nothing can be classified as superseded, so leave + // every entry alone rather than guess. + if (liveIndex < 0) return; + + const reclaimedPaths: string[] = []; + let failed = 0; + const survivors: IndexGcLedger["generations"] = []; + for (const [index, entry] of ledger.generations.entries()) { + // A malformed entry is kept, never iterated. This also runs on the load + // path, where a throw would null the loaded index and trigger the + // full-corpus rebuild. A GC step must not be able to fail a load. + if (index >= liveIndex || !entry || !Array.isArray(entry.shards)) { + survivors.push(entry); + continue; + } + const stranded: IndexGcLedger["generations"][number]["shards"] = []; + for (const shard of entry.shards) { + try { + await this.kv.delete(shard.scope, shard.key); + reclaimedPaths.push(statePath(shard.scope, shard.key)); + } catch { + failed += 1; + stranded.push(shard); + } + } + if (stranded.length > 0) { + survivors.push({ generation: entry.generation, shards: stranded }); + } + } + + // One audit row for the sweep, not one per shard. src/functions/audit.ts + // sets the policy: automatic bulk sweeps emit a single row listing every + // removed id, because per-item rows flood the log. A reclaim on a badly + // leaked store is well over a thousand shards, and this runs during boot. + if (reclaimedPaths.length > 0) { + await this.auditIndexPersistence("delete", reclaimedPaths, { + manifestKey, + reason: "generation_reclaim", + liveGeneration, + // `evicted` is the field name src/functions/audit.ts specifies for a + // sweep; retention.ts is the reference shape. `failed` surfaces deletes + // that will be retried, which would otherwise vanish silently. + evicted: reclaimedPaths.length, + failed, + }); + } + + // A successful delete is the only thing that shrinks the ledger, so with + // none there is nothing to rewrite. Without this, every boot writes the + // ledger back unchanged. + if (reclaimedPaths.length === 0) return; + await this.kv + .set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations: survivors, + }) + .catch(() => undefined); + } + private async auditIndexPersistence( action: string, targetIds: string[], @@ -280,35 +613,41 @@ export class IndexPersistence { ); } + /** Reports whether the delete landed, so the reclaim path can retry the rest. */ private async deleteKey( scope: string, key: string, reason: string, - ): Promise { - let result = "deleted"; + ): Promise { + let ok = true; let error: string | undefined; try { await this.kv.delete(scope, key); } catch (err) { - result = "failed"; + ok = false; error = errorMessage(err); } await this.auditIndexPersistence("delete", [statePath(scope, key)], { scope, key, reason, - result, + result: ok ? "deleted" : "failed", error, }); + return ok; } private async deleteShards( shards: IndexShardManifest["shards"], reason: string, - ): Promise { + ): Promise { + let allGone = true; for (const shard of shards) { - await this.deleteKey(shard.scope, shard.key, reason); + if (!(await this.deleteKey(shard.scope, shard.key, reason))) { + allGone = false; + } } + return allGone; } private async isManifestPublished( @@ -368,7 +707,20 @@ export class IndexPersistence { manifest.value != null && typeof manifest.value === "object" ) { - return this.loadManifestData(manifest.value, label); + const data = await this.loadManifestData(manifest.value, label); + // Boot is the only point that sees a generation stranded by a kill: the + // save path only ever inspects its own predecessor. Reclaim once the live + // generation has actually loaded — a failed load must not authorise + // deleting anything. + if (data !== null) { + // Through the same queue as save(), or this sweep's ledger rewrite + // clobbers a concurrent save's entry. Never allowed to fail the load. + const live = manifest.value.generation; + await this + .enqueue(() => this.reclaimGenerations(manifestKey, live)) + .catch(() => undefined); + } + return data; } const legacy = await this.readIndexValue( diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 929791657f..3dff054091 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -563,6 +563,383 @@ describe("IndexPersistence", () => { expect(loaded.bm25!.search("alpha").length).toBe(0); }); + it("reclaims the previous generation when the previous manifest read fails (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // The manifest read that opens saveShardedIndex times out. It used to be + // swallowed into `previous = null`, which skipped the cleanup guard and + // stranded gen_old's shards with nothing left to ever revisit them. + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(readFailsKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("reclaims a generation stranded by a failed cleanup on the next load (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const cleanupKv = { + ...kv, + delete: vi.fn(async () => { + throw new Error("cleanup failed"); + }), + }; + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(cleanupKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + // Cleanup failed, so gen_old is still on disk. Nothing in the save path + // will revisit it — a later save only ever inspects its own predecessor. + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25!.search("bravo").length).toBe(1); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("reclaims the previous vector generation when the vector manifest read fails (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + makeVector("obs_old"), + { shardChars: 80, createGeneration: () => "gen_old" }, + ).save(); + const oldVectorScope = "mem:index:bm25:vectors:gen_old:00000"; + await expect(kv.get(oldVectorScope, "data")).resolves.not.toBeNull(); + + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + readFailsKv as never, + makeBm25("obs_new", "bravo new snapshot"), + makeVector("obs_new"), + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(oldVectorScope, "data")).resolves.toBeNull(); + }); + + it("leaves a generation recorded after the live one untouched on load (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_live" }, + ).save(); + + // A concurrent save has recorded its generation and is mid-write, but has + // not published its manifest yet. setIndexPersistence runs before load() + // in src/index.ts, so a request arriving during boot produces exactly this. + const inflightScope = "mem:index:bm25:bm25:gen_inflight:00000"; + await kv.set(inflightScope, "data", "partial shard"); + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; + }>(BM25_SCOPE, gcKey); + ledger!.generations.push({ + generation: "gen_inflight", + shards: [{ scope: inflightScope, key: "data" }], + }); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + + // Deleting it would leave the in-flight save publishing a manifest whose + // shards are already gone, which fails closed on the next load. + await expect(kv.get(inflightScope, "data")).resolves.toBe("partial shard"); + }); + + it("reclaims a pre-ledger manifest that carries no generation (#1115)", async () => { + const legacyScope = "mem:index:bm25:bm25:gen_legacy:00000"; + await kv.set(legacyScope, "data", "legacy shard"); + await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { + v: 1, + shards: [{ scope: legacyScope, key: "data", chars: 12 }], + chars: 12, + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(legacyScope, "data")).resolves.toBeNull(); + }); + + it("keeps a published manifest whole when two saves overlap (#1115)", async () => { + // scheduleSave fires save() unawaited from a timer while flushIndexSave + // awaits save() on every delete path, so two saves on ONE instance is the + // normal shape, not a contrivance. Without the queue, whichever published + // second reclaimed the other's shards and left it naming data already gone. + vi.useRealTimers(); + const store = new Map>(); + const slowKv = { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + // Stall one of the first generation's shard writes so the second save + // overtakes it. + if (scope.includes(":gen_0:") && scope.endsWith("00005")) { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from((store.get(scope)?.values() ?? []) as Iterable), + }; + + const bm25 = new SearchIndex(); + for (let i = 0; i < 30; i++) { + bm25.add( + makeObs({ id: `obs_${i}`, title: `lorem ipsum dolor sit amet ${i}` }), + ); + } + let generation = 0; + const persistence = new IndexPersistence(slowKv as never, bm25, null, { + shardChars: 400, + createGeneration: () => `gen_${generation++}`, + }); + + // Let the first save get into its shard writes before the second starts, + // so the second is the one that publishes. + const first = persistence.save(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const second = persistence.save(); + await Promise.all([first, second]); + + const manifest = await slowKv.get( + BM25_SCOPE, + BM25_MANIFEST_KEY, + ); + const missing: string[] = []; + for (const shard of manifest!.shards) { + if ((await slowKv.get(shard.scope, shard.key)) === null) { + missing.push(shard.scope); + } + } + expect(missing).toEqual([]); + + const loaded = await new IndexPersistence( + slowKv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25).not.toBeNull(); + }); + + it("still persists the index when the gc ledger is unreadable (#1115)", async () => { + // state::get timing out is the condition this bug appears under. Aborting + // the save there would stop persisting entirely, which is worse than the + // leak: a lost BM25 index costs a full-corpus rebuild. + const failingKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still persists when the gc ledger holds a malformed entry (#1115)", async () => { + // A null entry used to throw a TypeError out of saveShardedIndex before any + // shard write, leaving one throttled log line per 60s while BM25 silently + // stopped persisting for good. + await kv.set(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`, { + v: 1, + generations: [null], + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still reclaims when the gc ledger holds a malformed entry (#1115)", async () => { + // Publishing is not enough. A malformed entry that silently disabled + // reclaim would leak a generation every cycle while this test stayed green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ v: 1; generations: unknown[] }>( + BM25_SCOPE, + gcKey, + ); + ledger!.generations.unshift(null); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, makeBm25("obs_new", "bravo"), null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("falls back to previous-generation cleanup when the ledger is unusable (#1115)", async () => { + // Measured regression guard: with no fallback, an unusable ledger leaked + // one generation per cycle — strictly worse than the code this replaced, + // which always had this path. Without this test the `else if` can be + // deleted with the whole suite green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // Ledger read fails, manifest read succeeds. Both are state::get with + // independent timeouts, so this split is ordinary, not contrived. + const ledgerFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + ledgerFailsKv as never, + makeBm25("obs_new", "bravo"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("keeps the ledger entry when a rollback delete fails (#1115)", async () => { + // Shards that survived their rollback delete must keep the entry that + // names them, or nothing can ever reclaim them. + let shardWrites = 0; + const failingKv = { + ...kv, + set: vi.fn(async (scope: string, key: string, data: T): Promise => { + if (scope.includes(":gen_new:")) { + shardWrites += 1; + if (shardWrites === 2) throw new Error("shard write failed"); + } + return kv.set(scope, key, data); + }), + delete: vi.fn(async () => { + throw new Error("rollback delete failed"); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const ledger = await kv.get<{ + v: 1; + generations: Array<{ generation: string }>; + }>(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`); + expect(ledger!.generations.map((entry) => entry.generation)).toContain( + "gen_new", + ); + }); + + it("refuses to overwrite a gc ledger it does not recognise (#1115)", async () => { + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const future = { v: 2, generations: [], writtenByANewerBuild: true }; + await kv.set(BM25_SCOPE, gcKey, future); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + // Rewriting it as a v1 ledger would drop every generation it tracked. + await expect(kv.get(BM25_SCOPE, gcKey)).resolves.toEqual(future); + }); + it("keeps the previous vector generation when vector save fails after BM25 publish", async () => { const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); const previousVector = makeVector("obs_old"); From 25045d83db500d307dcc1d79869213d8cc5a2a64 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 21:07:58 +0800 Subject: [PATCH 13/27] fix(providers): bound provider concurrency and stop counting 429s as failures ResilientProvider wrapped the LLM provider with a circuit breaker and nothing else. There was no semaphore, no queue and no concurrency ceiling anywhere on the provider path, so every captured observation fired compress() immediately. Under multi-subagent load that is dozens of simultaneous upstream calls, and the provider answered exactly as you would expect: OpenAI API error (429): {"error":"too many concurrent requests"} Those 429s were then fed to the breaker. At the configured threshold of three failures in sixty seconds it opened, and for the next thirty seconds every compression failed fast with circuit_breaker_open while the provider was perfectly healthy and merely busy. Observed in production: continuous circuit_breaker_open on compression, so observations were captured but never compressed. Two changes, and the second matters as much as the first. Bound the number of in-flight calls, default four and overridable per instance or with AGENTMEMORY_MAX_PROVIDER_CONCURRENCY. The semaphore hands a released slot directly to the next waiter instead of decrementing and letting waiters race for the opening, and releases in a finally so a throw cannot leak a slot and deadlock everything queued behind it. Stop recording a rate-limit response as a breaker failure. A 429 says slow down, not that the provider is broken; treating backpressure as a fault is what converted a busy upstream into a thirty-second outage. Genuine failures still open the breaker, and the breaker is still checked before queueing so an open circuit fails fast rather than occupying a slot. Tests cover peak concurrency under a burst of 24 calls, slot release on the error path, that 429s leave the breaker closed, and that real errors still open it. Each was run against the unmodified source first: peak concurrency measured 24 against a limit of 3, and the breaker measured open where it should have been closed. --- src/providers/circuit-breaker.ts | 2 +- src/providers/resilient.ts | 84 ++++++++++++++++++++- test/resilient-provider.test.ts | 125 +++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 test/resilient-provider.test.ts diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index e7f1f85d23..b6132662a8 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -1,6 +1,6 @@ import type { CircuitBreakerState } from "../types.js"; -interface CircuitBreakerOptions { +export interface CircuitBreakerOptions { failureThreshold?: number; failureWindowMs?: number; recoveryTimeoutMs?: number; diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 95ece40c96..a2c51151fc 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,25 +1,101 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; -import { CircuitBreaker } from "./circuit-breaker.js"; +import { CircuitBreaker, type CircuitBreakerOptions } from "./circuit-breaker.js"; + +const DEFAULT_MAX_CONCURRENT = 4; + +/** + * Bounds how many calls are inside the provider at once. + * + * Without this, every captured observation fired a compress() immediately and + * unbounded, so load meant dozens of simultaneous upstream calls and the + * provider answered with `429 too many concurrent requests`. + * + * release() hands its slot directly to the next waiter rather than decrementing + * and letting it race for the opening, so the limit holds under a burst. + */ +class Semaphore { + private active = 0; + private waiting: Array<() => void> = []; + + constructor(private readonly limit: number) {} + + async acquire(): Promise { + if (this.active < this.limit) { + this.active++; + return; + } + await new Promise((resolve) => this.waiting.push(resolve)); + } + + release(): void { + const next = this.waiting.shift(); + if (next) next(); + else this.active--; + } +} + +/** + * A 429 says "slow down", not "you are broken". + * + * Counting it as a circuit-breaker failure turns backpressure into an outage: + * three of them inside the failure window open the breaker, and then every + * compression fails fast for the whole recovery timeout even though the + * provider is healthy and merely busy. + */ +function isRateLimited(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return ( + /\b429\b/.test(message) || + /rate.?limit|too many (concurrent )?requests/i.test(message) + ); +} + +function resolveMaxConcurrent(configured: number | undefined): number { + const candidate = + configured ?? + Number(process.env["AGENTMEMORY_MAX_PROVIDER_CONCURRENCY"] ?? NaN); + if (!Number.isFinite(candidate)) return DEFAULT_MAX_CONCURRENT; + const whole = Math.floor(candidate); + return whole >= 1 ? whole : DEFAULT_MAX_CONCURRENT; +} + +export interface ResilientProviderOptions { + maxConcurrent?: number; + breaker?: CircuitBreakerOptions; +} export class ResilientProvider implements MemoryProvider { - private breaker = new CircuitBreaker(); + private breaker: CircuitBreaker; + private gate: Semaphore; name: string; - constructor(private inner: MemoryProvider) { + constructor( + private inner: MemoryProvider, + options: ResilientProviderOptions = {}, + ) { + this.breaker = new CircuitBreaker(options.breaker); + this.gate = new Semaphore(resolveMaxConcurrent(options.maxConcurrent)); this.name = `resilient(${inner.name})`; } private async call(fn: () => Promise): Promise { + // Checked before queueing. An open breaker should fail fast rather than + // occupy a slot that a call with a chance of succeeding could use. if (!this.breaker.isAllowed) { throw new Error("circuit_breaker_open"); } + await this.gate.acquire(); try { const result = await fn(); this.breaker.recordSuccess(); return result; } catch (err) { - this.breaker.recordFailure(); + if (!isRateLimited(err)) this.breaker.recordFailure(); throw err; + } finally { + // In `finally` so a throw cannot leak the slot and deadlock every call + // queued behind it. + this.gate.release(); } } diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts new file mode 100644 index 0000000000..70dd9aa705 --- /dev/null +++ b/test/resilient-provider.test.ts @@ -0,0 +1,125 @@ +import { describe, it, expect } from "vitest"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +// Tracks how many calls are inside the provider at once, which is the thing the +// upstream API actually rejects. Counting total calls would not discriminate. +function countingProvider( + behaviour: (n: number) => Promise = async () => "ok", +): MemoryProvider & { peak: number; calls: number } { + let inFlight = 0; + const state = { + name: "counting", + peak: 0, + calls: 0, + async compress(): Promise { + inFlight++; + state.calls++; + if (inFlight > state.peak) state.peak = inFlight; + try { + return await behaviour(state.calls); + } finally { + inFlight--; + } + }, + async summarize(): Promise { + return state.compress(); + }, + }; + return state; +} + +function rateLimited(): Error { + return new Error('OpenAI API error (429): {"error":"too many concurrent requests"}'); +} + +describe("ResilientProvider concurrency", () => { + it("never runs more than the configured number of calls at once", async () => { + const inner = countingProvider( + () => new Promise((resolve) => setTimeout(() => resolve("ok"), 5)), + ); + const provider = new ResilientProvider(inner, { maxConcurrent: 3 }); + + await Promise.all( + Array.from({ length: 24 }, () => provider.compress("sys", "user")), + ); + + // The provider fired all 24 at once before this bound existed, which is + // exactly what the upstream 429 complains about. + expect(inner.peak).toBeLessThanOrEqual(3); + expect(inner.calls).toBe(24); + }); + + it("still completes every call while bounded", async () => { + const inner = countingProvider(); + const provider = new ResilientProvider(inner, { maxConcurrent: 2 }); + + const results = await Promise.all( + Array.from({ length: 10 }, () => provider.compress("sys", "user")), + ); + + expect(results).toHaveLength(10); + expect(results.every((r) => r === "ok")).toBe(true); + }); + + it("releases its slot when a call throws", async () => { + const inner = countingProvider(async (n) => { + if (n <= 2) throw new Error("boom"); + return "ok"; + }); + const provider = new ResilientProvider(inner, { + maxConcurrent: 1, + breaker: { failureThreshold: 100 }, + }); + + const settled = await Promise.allSettled( + Array.from({ length: 5 }, () => provider.compress("sys", "user")), + ); + + // A slot leaked on the error path would deadlock the remaining calls. + expect(settled).toHaveLength(5); + expect(settled.filter((s) => s.status === "fulfilled")).toHaveLength(3); + }); +}); + +describe("ResilientProvider rate limiting", () => { + it("does not open the breaker on 429s", async () => { + // A 429 means slow down, not that the provider is broken. Counting it as a + // breaker failure converts backpressure into a 30s outage in which every + // compression fails fast. + const inner = countingProvider(async (n) => { + if (n <= 5) throw rateLimited(); + return "ok"; + }); + const provider = new ResilientProvider(inner, { + maxConcurrent: 1, + breaker: { failureThreshold: 3 }, + }); + + for (let i = 0; i < 5; i++) { + await provider.compress("sys", "user").catch(() => undefined); + } + + expect(provider.circuitState.state).toBe("closed"); + await expect(provider.compress("sys", "user")).resolves.toBe("ok"); + }); + + it("still opens the breaker on genuine failures", async () => { + const inner = countingProvider(async () => { + throw new Error("upstream exploded"); + }); + const provider = new ResilientProvider(inner, { + maxConcurrent: 1, + breaker: { failureThreshold: 3 }, + }); + + for (let i = 0; i < 3; i++) { + await provider.compress("sys", "user").catch(() => undefined); + } + + expect(provider.circuitState.state).toBe("open"); + await expect(provider.compress("sys", "user")).rejects.toThrow( + "circuit_breaker_open", + ); + }); +}); From f5c822f038f283d5ddeaf3137dbc4ed68d8c2df2 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 21:32:09 +0800 Subject: [PATCH 14/27] refactor(providers): read the concurrency limit through getEnvVar Reading the process environment directly bypassed the user-level agentmemory config file, which is the surface every sibling provider goes through (minimax.ts, providers/index.ts and _fetch.ts all use getEnvVar). Same line count, and it closes a real gap: an operator setting the limit there would have seen it silently ignored. Drop the breaker options passthrough. Nothing in production supplies it, all three constructions in providers/index.ts are bare, and every test that passed it was restating a default. CircuitBreakerOptions goes back to being private. Collapse the one-field options interface into an inline type, merge the two rate-limit regexes into one with an identical truth table, and cut the comments that restated the commit message. The release-handoff note stays: it is design rationale, not narrative. Fold the completion assertion into the concurrency test, which already proved it by resolving all 24 calls, and drop the maxConcurrent arguments that were no-ops against sequentially awaited calls. The one in the slot-release test stays, because at the default of 4 a leaked slot would not deadlock and that test would pass despite the bug. --- src/providers/circuit-breaker.ts | 2 +- src/providers/resilient.ts | 41 +++++++++++--------------------- test/resilient-provider.test.ts | 38 +++++++---------------------- 3 files changed, 23 insertions(+), 58 deletions(-) diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index b6132662a8..e7f1f85d23 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -1,6 +1,6 @@ import type { CircuitBreakerState } from "../types.js"; -export interface CircuitBreakerOptions { +interface CircuitBreakerOptions { failureThreshold?: number; failureWindowMs?: number; recoveryTimeoutMs?: number; diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index a2c51151fc..dd6c4dc223 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,17 +1,15 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; -import { CircuitBreaker, type CircuitBreakerOptions } from "./circuit-breaker.js"; +import { CircuitBreaker } from "./circuit-breaker.js"; +import { getEnvVar } from "../config.js"; const DEFAULT_MAX_CONCURRENT = 4; /** * Bounds how many calls are inside the provider at once. * - * Without this, every captured observation fired a compress() immediately and - * unbounded, so load meant dozens of simultaneous upstream calls and the - * provider answered with `429 too many concurrent requests`. - * - * release() hands its slot directly to the next waiter rather than decrementing - * and letting it race for the opening, so the limit holds under a burst. + * release() hands its slot directly to the next waiter rather than + * decrementing and letting waiters race for the opening, so the limit holds + * under a burst. */ class Semaphore { private active = 0; @@ -34,36 +32,25 @@ class Semaphore { } } -/** - * A 429 says "slow down", not "you are broken". - * - * Counting it as a circuit-breaker failure turns backpressure into an outage: - * three of them inside the failure window open the breaker, and then every - * compression fails fast for the whole recovery timeout even though the - * provider is healthy and merely busy. - */ +// 429 is backpressure, not a fault. Counting it opened the breaker on a healthy +// provider and failed every compression for the recovery window. function isRateLimited(err: unknown): boolean { const message = err instanceof Error ? err.message : String(err); - return ( - /\b429\b/.test(message) || - /rate.?limit|too many (concurrent )?requests/i.test(message) - ); + return /\b429\b|rate.?limit|too many (concurrent )?requests/i.test(message); } +// Option is the test seam, env is the operator knob on Railway, constant is the +// default. getEnvVar rather than process.env so ~/.agentmemory/.env is honoured, +// which is how every sibling provider reads config. function resolveMaxConcurrent(configured: number | undefined): number { const candidate = configured ?? - Number(process.env["AGENTMEMORY_MAX_PROVIDER_CONCURRENCY"] ?? NaN); + Number(getEnvVar("AGENTMEMORY_MAX_PROVIDER_CONCURRENCY") ?? NaN); if (!Number.isFinite(candidate)) return DEFAULT_MAX_CONCURRENT; const whole = Math.floor(candidate); return whole >= 1 ? whole : DEFAULT_MAX_CONCURRENT; } -export interface ResilientProviderOptions { - maxConcurrent?: number; - breaker?: CircuitBreakerOptions; -} - export class ResilientProvider implements MemoryProvider { private breaker: CircuitBreaker; private gate: Semaphore; @@ -71,9 +58,9 @@ export class ResilientProvider implements MemoryProvider { constructor( private inner: MemoryProvider, - options: ResilientProviderOptions = {}, + options: { maxConcurrent?: number } = {}, ) { - this.breaker = new CircuitBreaker(options.breaker); + this.breaker = new CircuitBreaker(); this.gate = new Semaphore(resolveMaxConcurrent(options.maxConcurrent)); this.name = `resilient(${inner.name})`; } diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts index 70dd9aa705..4f7b028a13 100644 --- a/test/resilient-provider.test.ts +++ b/test/resilient-provider.test.ts @@ -40,25 +40,13 @@ describe("ResilientProvider concurrency", () => { ); const provider = new ResilientProvider(inner, { maxConcurrent: 3 }); - await Promise.all( + const results = await Promise.all( Array.from({ length: 24 }, () => provider.compress("sys", "user")), ); - // The provider fired all 24 at once before this bound existed, which is - // exactly what the upstream 429 complains about. expect(inner.peak).toBeLessThanOrEqual(3); + // Bounding must not drop work: every call still ran and still resolved. expect(inner.calls).toBe(24); - }); - - it("still completes every call while bounded", async () => { - const inner = countingProvider(); - const provider = new ResilientProvider(inner, { maxConcurrent: 2 }); - - const results = await Promise.all( - Array.from({ length: 10 }, () => provider.compress("sys", "user")), - ); - - expect(results).toHaveLength(10); expect(results.every((r) => r === "ok")).toBe(true); }); @@ -67,16 +55,14 @@ describe("ResilientProvider concurrency", () => { if (n <= 2) throw new Error("boom"); return "ok"; }); - const provider = new ResilientProvider(inner, { - maxConcurrent: 1, - breaker: { failureThreshold: 100 }, - }); + // maxConcurrent 1 is load-bearing here: at the default of 4 a leaked slot + // would not deadlock, and the test would pass despite the bug. + const provider = new ResilientProvider(inner, { maxConcurrent: 1 }); const settled = await Promise.allSettled( Array.from({ length: 5 }, () => provider.compress("sys", "user")), ); - // A slot leaked on the error path would deadlock the remaining calls. expect(settled).toHaveLength(5); expect(settled.filter((s) => s.status === "fulfilled")).toHaveLength(3); }); @@ -84,18 +70,13 @@ describe("ResilientProvider concurrency", () => { describe("ResilientProvider rate limiting", () => { it("does not open the breaker on 429s", async () => { - // A 429 means slow down, not that the provider is broken. Counting it as a - // breaker failure converts backpressure into a 30s outage in which every - // compression fails fast. const inner = countingProvider(async (n) => { if (n <= 5) throw rateLimited(); return "ok"; }); - const provider = new ResilientProvider(inner, { - maxConcurrent: 1, - breaker: { failureThreshold: 3 }, - }); + const provider = new ResilientProvider(inner); + // Five, against a default failure threshold of three. for (let i = 0; i < 5; i++) { await provider.compress("sys", "user").catch(() => undefined); } @@ -108,10 +89,7 @@ describe("ResilientProvider rate limiting", () => { const inner = countingProvider(async () => { throw new Error("upstream exploded"); }); - const provider = new ResilientProvider(inner, { - maxConcurrent: 1, - breaker: { failureThreshold: 3 }, - }); + const provider = new ResilientProvider(inner); for (let i = 0; i < 3; i++) { await provider.compress("sys", "user").catch(() => undefined); From b8656b042385d2ce68fdf86e9746b271b15028b3 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 23:53:03 +0800 Subject: [PATCH 15/27] fix(mcp): read graph stats from the snapshot instead of enumerating the graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agentmemory://graph/stats listed every node and every edge in order to produce four aggregate numbers. On a real corpus that is a multi-hundred-megabyte frame over the engine transport, against the 16 MiB limit this repo already defines in state/frame-guard.ts. Measured on a deployed instance: the two scopes hold 112 MB and 80 MB, five and seven times the limit, so the read cannot succeed — it closes the worker connection with 1009 Message Too Big every time. The observed rate was 3.8 to 5.1 kills per minute, each followed by a reconnect that registers a fresh worker id. The counts were already precomputed. KV.graphSnapshot carries totalNodes, totalEdges, nodesByType and edgesByType in a single small key, and the comment introducing it in state/schema.ts names /graph/stats as the caller it was built to spare. This resource simply never adopted it. An absent snapshot now reports zeros with pending: true rather than falling back to enumeration. The fallback is the failure mode, so there is no version of it worth keeping. Other enumerating callers remain and need separate treatment: cascade.ts needs a sourceObservationId index before it can stop scanning, and the export route in api.ts guards its response size but not the inbound read that already happened. --- src/mcp/server.ts | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ef26427aaf..4195941faf 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -5,8 +5,7 @@ import type { SessionSummary, Memory, Session, - GraphNode, - GraphEdge, + GraphSnapshot, } from "../types.js"; import { getVisibleTools } from "./tools-registry.js"; import { timingSafeCompare } from "../auth.js"; @@ -1479,14 +1478,18 @@ export function registerMcpEndpoints( if (uri === "agentmemory://graph/stats") { try { - const nodes = await kv.list(KV.graphNodes); - const edges = await kv.list(KV.graphEdges); - const nodesByType: Record = {}; - for (const n of nodes) - nodesByType[n.type] = (nodesByType[n.type] || 0) + 1; - const edgesByType: Record = {}; - for (const e of edges) - edgesByType[e.type] = (edgesByType[e.type] || 0) + 1; + // Read the precomputed snapshot rather than enumerating the graph. + // KV.graphSnapshot exists for exactly this (#814, state/schema.ts): + // it carries the same aggregate counts in one small key. Listing + // the scopes instead pulled every node and edge over the engine + // transport — on a real corpus that is a multi-hundred-MB frame + // against a 16 MiB limit (state/frame-guard.ts), which kills the + // worker connection every time this resource is read. + const snapshot = await kv.get( + KV.graphSnapshot, + "current", + ); + const stats = snapshot?.stats; return { status_code: 200, body: { @@ -1495,10 +1498,13 @@ export function registerMcpEndpoints( uri, mimeType: "application/json", text: JSON.stringify({ - totalNodes: nodes.length, - totalEdges: edges.length, - nodesByType, - edgesByType, + totalNodes: stats?.totalNodes ?? 0, + totalEdges: stats?.totalEdges ?? 0, + nodesByType: stats?.nodesByType ?? {}, + edgesByType: stats?.edgesByType ?? {}, + // Absent snapshot reports zeros rather than falling back + // to a scope enumeration: the fallback is the failure. + ...(snapshot ? {} : { pending: true }), }), }, ], From 26ef3ed2c117ba8c93072a53fd3220bf815ac20a Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 00:50:33 +0800 Subject: [PATCH 16/27] docs(mcp): drop what-narrating comments from the graph stats read AGENTS.md coding standards forbid comments that explain what code does. KV.graphSnapshot and the `pending: true` spread already say what the two blocks were narrating, so they carried no information the names do not. The rationale is not lost. The frame-limit failure mode and the deliberate refusal to fall back to enumeration are both recorded in the parent commit's message and in the PR body, which is where a reader looking for why will go. Comment-only: nine lines removed, none added, no executable statement touched. --- src/mcp/server.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 4195941faf..88142ea5d5 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1478,13 +1478,6 @@ export function registerMcpEndpoints( if (uri === "agentmemory://graph/stats") { try { - // Read the precomputed snapshot rather than enumerating the graph. - // KV.graphSnapshot exists for exactly this (#814, state/schema.ts): - // it carries the same aggregate counts in one small key. Listing - // the scopes instead pulled every node and edge over the engine - // transport — on a real corpus that is a multi-hundred-MB frame - // against a 16 MiB limit (state/frame-guard.ts), which kills the - // worker connection every time this resource is read. const snapshot = await kv.get( KV.graphSnapshot, "current", @@ -1502,8 +1495,6 @@ export function registerMcpEndpoints( totalEdges: stats?.totalEdges ?? 0, nodesByType: stats?.nodesByType ?? {}, edgesByType: stats?.edgesByType ?? {}, - // Absent snapshot reports zeros rather than falling back - // to a scope enumeration: the fallback is the failure. ...(snapshot ? {} : { pending: true }), }), }, From 20b1a24a18b0faf45e2d73fe999e7d2ae5dbc9a7 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:45:59 +0800 Subject: [PATCH 17/27] docs(state): drop the review marker from the gc ledger append The `ponytail:` prefix marks an internal review pass and has no business in an upstream PR. The same marker was already removed from the #1115 branch; a cherry-pick duplicated it onto this one, so that removal never reached here. The limitation it recorded is real and moves to the PR body: while every shard delete keeps failing, each debounce appends a generation carrying all its shard descriptors, and reclaimGenerations only rewrites the ledger when something was actually reclaimed, so the ledger grows unbounded until a delete succeeds. Reaching it needs a total delete outage. A length cap is the upgrade path if it ever shows up in the wild. Comment-only. No behaviour change. --- src/state/index-persistence.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 8b0d125423..bfe97a72cb 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -501,11 +501,6 @@ export class IndexPersistence { }); } - // ponytail: unbounded while EVERY delete keeps failing — each debounce - // appends a generation carrying all its shard descriptors, and - // reclaimGenerations only rewrites when something was actually reclaimed. - // Add a length cap if that ever shows up in the wild; a cap is not one - // line, and this needs a total delete outage to reach. await this.kv.set( KV.bm25Index, this.gcKey(manifestKey), From 76db640d34f461f61d4e3cb8597258af38e4bbba Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:57:07 +0800 Subject: [PATCH 18/27] feat(cli): forward the engine's own stdout and stderr, bounded and opt-in Rebased onto the deploy branch, which carries 8fd1c32 "exit when the iii engine dies after startup" and forked before the upstream cli.ts rewrite, so the spawn region differs from the upstream form of this change. `spawnEngineBackground` launched the engine with `stdio: ["ignore", "ignore", "pipe"]`. The engine's stdout was discarded outright and its stderr only ever reached the 16 KiB `stderrChunks` capture, which exists to quote the dying engine in the death report and is never forwarded. Every `[iii]` line visible in a deployment therefore comes from the SDK inside node, never from the engine. That matters because node's side of a `WS_ERR_UNSUPPORTED_MESSAGE_LENGTH` kill carries no attribution: the `ws` Receiver rejects the frame at the 64-bit length header, so no handler runs, no `function_id` is recorded, and the error text is character-identical whichever scope produced the frame. The engine is the only process that knows which invocation it was serialising, and the only place `Worker unregistered` could ever appear. Forwarding is gated on the AGENTMEMORY_ENGINE_LOG environment variable and off by default. With it off, the stdio array is the previous `"ignore"` and behaviour is unchanged. With it on, stdout is piped and both streams are line-split, prefixed `[engine]` and `[engine:err]`, and written to the process log. Engine-death detection is deliberately untouched. It keys off `child.on("exit")`, a process event that does not depend on stdio being consumed, and it reads `stderrChunks`, filled by its own listener. The forwarder adds a second `data` listener rather than replacing that one, and node delivers each chunk to every listener, so the capture still fills byte for byte. Measured on a real child that writes to both streams and dies: exit fires with the same code and the capture holds the same 39 bytes including the panic, with the flag on and with it off. Three further properties were measured rather than argued. A blocking writer pushing 8,388,608 bytes through a piped stdout drained completely with the forwarder already past its lifetime ceiling and discarding every line, so the ceiling never turns into a stalled reader that wedges the engine on a full pipe. The death report still reaches the log with 65,602 bytes of forwarder output queued ahead of it, at every reader throttle tested and matching a no-forwarder control. And the forwarder writes only to the process log while the engine's stdin stays `"ignore"`, so no forwarded line can re-enter the forwarder. An unbounded log path here already caused an incident (#519, 137 GB of `daemon.log` for one user, from a lag warning that re-entered the stream it was warning about), so the forwarder is capped at 64 KiB/s peak and 32 MiB for the lifetime of the process. At the lifetime ceiling it writes one terminal line and goes permanently silent while still draining. Over the rate cap it counts withheld bytes and reports them once per window; that notice is exempt from the rate cap alone, so suppression can never be silent. `test/cli-engine-startup.test.ts` does not exist on this branch and its upstream form asserts against helpers this branch does not have, so it is added here with the assertions that apply: the stdio gate, and guards pinning that the death-detection state, its stderr listener and its exit handler all stay outside the forwarder's gate. 8fd1c32 shipped without tests, so those guards are new coverage for it as well. --- src/cli.ts | 29 +++++- src/cli/engine-log.ts | 112 ++++++++++++++++++++++ test/cli-engine-startup.test.ts | 58 ++++++++++++ test/engine-log.test.ts | 159 ++++++++++++++++++++++++++++++++ 4 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 src/cli/engine-log.ts create mode 100644 test/cli-engine-startup.test.ts create mode 100644 test/engine-log.test.ts diff --git a/src/cli.ts b/src/cli.ts index b965782044..6474a42949 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -54,6 +54,7 @@ import { type ConnectManifest, type RemoveOptions, } from "./cli/remove-plan.js"; +import { createEngineLogForwarder } from "./cli/engine-log.js"; import { renderSplash } from "./cli/splash.js"; import { isFirstRun, readPrefs, resetPrefs, writePrefs } from "./cli/preferences.js"; import { runOnboarding } from "./cli/onboarding.js"; @@ -917,6 +918,28 @@ type StartupFailure = { let startupFailure: StartupFailure | null = null; +// Off by default. The engine's own stdout is the only place a frame's +// `function_id` exists, but an unbounded log path here once wrote 137 GB +// (issue #519), so the forwarder is opt-in and hard-capped. +const ENGINE_LOG_ENABLED = + process.env["AGENTMEMORY_ENGINE_LOG"] === "1" || + process.env["AGENTMEMORY_ENGINE_LOG"] === "true"; + +function attachEngineLog( + stream: NodeJS.ReadableStream | null, + prefix: string, +): void { + if (!stream) return; + const forwarder = createEngineLogForwarder({ + prefix, + write: (line) => { + process.stderr.write(`${line}\n`); + }, + }); + stream.on("data", (chunk: Buffer) => forwarder.push(chunk)); + stream.on("end", () => forwarder.flush()); +} + // Spawn a background engine and collect any startup stderr for a short // window. The process is unref'd so the CLI parent can exit cleanly; we // only care about stderr that shows up BEFORE the health check succeeds, @@ -929,13 +952,17 @@ function spawnEngineBackground( vlog(`spawn: ${bin} ${spawnArgs.join(" ")}`); const child = spawn(bin, spawnArgs, { detached: true, - stdio: ["ignore", "ignore", "pipe"], + stdio: ["ignore", ENGINE_LOG_ENABLED ? "pipe" : "ignore", "pipe"], windowsHide: true, }); const isDocker = label.includes("Docker"); if (!isDocker && typeof child.pid === "number") { writeEnginePidfile(child.pid); } + if (ENGINE_LOG_ENABLED) { + attachEngineLog(child.stdout, "[engine]"); + attachEngineLog(child.stderr, "[engine:err]"); + } const spawnedAt = Date.now(); const stderrChunks: Buffer[] = []; let stderrBytes = 0; diff --git a/src/cli/engine-log.ts b/src/cli/engine-log.ts new file mode 100644 index 0000000000..2dab376e05 --- /dev/null +++ b/src/cli/engine-log.ts @@ -0,0 +1,112 @@ +import { StringDecoder } from "node:string_decoder"; + +export const ENGINE_LOG_MAX_TOTAL_BYTES = 32 * 1024 * 1024; +export const ENGINE_LOG_MAX_BYTES_PER_SECOND = 64 * 1024; +export const ENGINE_LOG_MAX_LINE_BYTES = 8 * 1024; +export const ENGINE_LOG_WINDOW_MS = 1000; + +export interface EngineLogForwarder { + push(chunk: Buffer): void; + flush(): void; +} + +export interface EngineLogForwarderOptions { + prefix: string; + write: (line: string) => void; + maxTotalBytes?: number; + maxBytesPerSecond?: number; + maxLineBytes?: number; + windowMs?: number; + now?: () => number; +} + +export function createEngineLogForwarder( + options: EngineLogForwarderOptions, +): EngineLogForwarder { + const { prefix, write } = options; + const maxTotalBytes = options.maxTotalBytes ?? ENGINE_LOG_MAX_TOTAL_BYTES; + const maxBytesPerSecond = + options.maxBytesPerSecond ?? ENGINE_LOG_MAX_BYTES_PER_SECOND; + const maxLineBytes = options.maxLineBytes ?? ENGINE_LOG_MAX_LINE_BYTES; + const windowMs = options.windowMs ?? ENGINE_LOG_WINDOW_MS; + const now = options.now ?? Date.now; + + const decoder = new StringDecoder("utf8"); + let pending = ""; + let totalBytes = 0; + let windowStart = now(); + let windowBytes = 0; + let droppedBytes = 0; + let stopped = false; + + function emitLine(line: string, exemptFromRateCap = false): void { + if (stopped) return; + const cost = Buffer.byteLength(line) + 1; + if (totalBytes + cost > maxTotalBytes) { + stopped = true; + write( + `${prefix} log forwarding stopped: ${maxTotalBytes} byte ceiling reached`, + ); + return; + } + if (!exemptFromRateCap && windowBytes + cost > maxBytesPerSecond) { + droppedBytes += cost; + return; + } + totalBytes += cost; + windowBytes += cost; + write(line); + } + + // The suppression notice is exempt from the rate cap so silent loss is + // impossible: it is one short line per window, still charged against + // both counters, so the lifetime ceiling still bounds it. + function reportDropped(): void { + if (droppedBytes === 0) return; + const dropped = droppedBytes; + droppedBytes = 0; + emitLine( + `${prefix} dropped ${dropped} bytes (rate cap ${maxBytesPerSecond} bytes/s)`, + true, + ); + } + + function rollWindow(): void { + if (now() - windowStart < windowMs) return; + windowStart = now(); + windowBytes = 0; + reportDropped(); + } + + function emitRecord(record: string): void { + emitLine(`${prefix} ${record.endsWith("\r") ? record.slice(0, -1) : record}`); + } + + return { + push(chunk: Buffer): void { + if (stopped) return; + rollWindow(); + pending += decoder.write(chunk); + let newline = pending.indexOf("\n"); + while (newline !== -1) { + emitRecord(pending.slice(0, newline)); + pending = pending.slice(newline + 1); + newline = pending.indexOf("\n"); + } + if (pending.length >= maxLineBytes) { + emitRecord(pending); + pending = ""; + } + }, + flush(): void { + if (stopped) return; + rollWindow(); + pending += decoder.end(); + if (pending.length > 0) { + emitRecord(pending); + pending = ""; + } + reportDropped(); + }, + }; +} diff --git a/test/cli-engine-startup.test.ts b/test/cli-engine-startup.test.ts new file mode 100644 index 0000000000..32551d7379 --- /dev/null +++ b/test/cli-engine-startup.test.ts @@ -0,0 +1,58 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("engine spawn stdio and death detection", () => { + const source = readFileSync("src/cli.ts", "utf8"); + const spawnStart = source.indexOf("function spawnEngineBackground"); + const spawnEnd = source.indexOf("const ENGINE_STARTUP_GRACE_MS", spawnStart); + const spawnBody = source.slice(spawnStart, spawnEnd); + + it("forwards engine stdout only behind the opt-in gate", () => { + expect(spawnBody).toContain( + 'stdio: ["ignore", ENGINE_LOG_ENABLED ? "pipe" : "ignore", "pipe"]', + ); + expect(spawnBody).not.toContain('stdio: ["ignore", "ignore", "pipe"]'); + expect(spawnBody).toContain( + 'if (ENGINE_LOG_ENABLED) {\n attachEngineLog(child.stdout, "[engine]");\n attachEngineLog(child.stderr, "[engine:err]");\n }', + ); + + expect(source).toContain( + 'const ENGINE_LOG_ENABLED =\n process.env["AGENTMEMORY_ENGINE_LOG"] === "1" ||\n process.env["AGENTMEMORY_ENGINE_LOG"] === "true"', + ); + expect(source).toContain('stream.on("end", () => forwarder.flush())'); + }); + + // Engine-death detection is the only thing that turns a dead engine into a + // container restart. Log forwarding shares the same spawn call and the same + // stderr stream, so these assertions exist to fail if forwarding is ever + // allowed to gate, replace, or reorder any part of it. + it("keeps engine-death detection ungated by the log forwarder", () => { + expect(spawnBody).toContain("const spawnedAt = Date.now()"); + expect(spawnBody).toContain("const stderrChunks: Buffer[] = []"); + expect(spawnBody).toContain( + 'child.stderr?.on("data", (chunk: Buffer) => {\n if (stderrBytes >= MAX_STDERR_CAPTURE) return;', + ); + expect(spawnBody).toContain("const engineRanFor = Date.now() - spawnedAt"); + expect(spawnBody).toContain("if (engineRanFor > ENGINE_STARTUP_GRACE_MS)"); + expect(spawnBody).toContain( + 'if (process.env["AGENTMEMORY_EXIT_ON_ENGINE_DEATH"] !== "0") {\n process.exit(1);', + ); + + // The capture listener and the exit handler must sit outside the gate. + const gateStart = spawnBody.indexOf("if (ENGINE_LOG_ENABLED) {"); + const gateEnd = spawnBody.indexOf("}", spawnBody.indexOf("[engine:err]")); + const gateBlock = spawnBody.slice(gateStart, gateEnd); + expect(gateBlock).not.toContain("stderrChunks"); + expect(gateBlock).not.toContain("process.exit(1)"); + expect(spawnBody.indexOf('child.on("exit"')).toBeGreaterThan(gateEnd); + }); + + it("still captures the dying engine's stderr for the death report", () => { + expect(spawnBody).toContain( + 'const stderr = Buffer.concat(stderrChunks).toString("utf-8")', + ); + expect(spawnBody).toContain( + "if (stderr.trim()) console.error(`[agentmemory] engine stderr:\\n${stderr}`)", + ); + }); +}); diff --git a/test/engine-log.test.ts b/test/engine-log.test.ts new file mode 100644 index 0000000000..6b839e1482 --- /dev/null +++ b/test/engine-log.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; +import { + ENGINE_LOG_MAX_BYTES_PER_SECOND, + ENGINE_LOG_MAX_TOTAL_BYTES, + createEngineLogForwarder, +} from "../src/cli/engine-log.js"; + +function harness(overrides: Record = {}) { + const lines: string[] = []; + let clock = 0; + const forwarder = createEngineLogForwarder({ + prefix: "[engine]", + write: (line) => lines.push(line), + now: () => clock, + ...overrides, + }); + return { + lines, + forwarder, + advance(ms: number) { + clock += ms; + }, + push(text: string) { + forwarder.push(Buffer.from(text, "utf8")); + }, + }; +} + +describe("engine log forwarder", () => { + it("prefixes each complete line and holds partial lines back", () => { + const h = harness(); + + h.push("registered worker\nWorker unregis"); + expect(h.lines).toEqual(["[engine] registered worker"]); + + h.push("tered\n"); + expect(h.lines).toEqual([ + "[engine] registered worker", + "[engine] Worker unregistered", + ]); + }); + + it("strips carriage returns and decodes utf8 split across chunks", () => { + const h = harness(); + const snowman = Buffer.from("☃", "utf8"); + + h.push("crlf line\r\n"); + h.forwarder.push(snowman.subarray(0, 1)); + h.forwarder.push(snowman.subarray(1)); + h.push("\n"); + + expect(h.lines).toEqual(["[engine] crlf line", "[engine] ☃"]); + }); + + it("flushes a trailing partial line when the stream ends", () => { + const h = harness(); + + h.push("panic: engine died"); + expect(h.lines).toEqual([]); + + h.forwarder.flush(); + expect(h.lines).toEqual(["[engine] panic: engine died"]); + }); + + it("stops permanently once the lifetime ceiling is reached", () => { + const h = harness({ maxTotalBytes: 64, maxBytesPerSecond: 1024 * 1024 }); + + for (let i = 0; i < 200; i += 1) h.push(`line ${i}\n`); + + const terminal = h.lines.filter((l) => l.includes("ceiling reached")); + expect(terminal).toHaveLength(1); + expect(terminal[0]).toBe( + "[engine] log forwarding stopped: 64 byte ceiling reached", + ); + expect(h.lines[h.lines.length - 1]).toBe(terminal[0]); + + const forwarded = h.lines.filter((l) => !l.includes("ceiling reached")); + const forwardedBytes = forwarded.reduce( + (sum, l) => sum + Buffer.byteLength(l) + 1, + 0, + ); + expect(forwardedBytes).toBeLessThanOrEqual(64); + + const before = h.lines.length; + h.push("still chatty\n"); + h.advance(10_000); + h.push("still chatty\n"); + h.forwarder.flush(); + expect(h.lines).toHaveLength(before); + }); + + it("stays silent for the rest of a single chunk that trips the ceiling", () => { + const h = harness({ maxTotalBytes: 64, maxBytesPerSecond: 1024 * 1024 }); + + h.push(Array.from({ length: 50 }, (_, i) => `line ${i}`).join("\n") + "\n"); + + expect(h.lines.filter((l) => l.includes("ceiling reached"))).toHaveLength(1); + expect(h.lines[h.lines.length - 1]).toBe( + "[engine] log forwarding stopped: 64 byte ceiling reached", + ); + }); + + it("drops over the rate cap and reports the drop once per window", () => { + const h = harness({ maxBytesPerSecond: 64, maxTotalBytes: 1024 * 1024 }); + const a = "a".repeat(30); + + h.push(`${a}\n`); + h.push(`${"b".repeat(30)}\n`); + h.push(`${"c".repeat(30)}\n`); + expect(h.lines).toEqual([`[engine] ${a}`]); + + h.advance(1000); + h.push(`${"d".repeat(30)}\n`); + expect(h.lines).toEqual([ + `[engine] ${a}`, + "[engine] dropped 80 bytes (rate cap 64 bytes/s)", + ]); + }); + + it("never suppresses silently, even with the window budget exhausted", () => { + const h = harness({ maxBytesPerSecond: 64, maxTotalBytes: 1024 * 1024 }); + + for (let i = 0; i < 20; i += 1) h.push(`${"a".repeat(30)}\n`); + h.forwarder.flush(); + + expect(h.lines).toEqual([ + `[engine] ${"a".repeat(30)}`, + "[engine] dropped 760 bytes (rate cap 64 bytes/s)", + ]); + }); + + it("charges its exempt notices against the lifetime ceiling", () => { + const h = harness({ maxBytesPerSecond: 8, maxTotalBytes: 200 }); + + for (let round = 0; round < 40; round += 1) { + h.push(`${"a".repeat(30)}\n`); + h.advance(1000); + } + + const forwardedBytes = h.lines + .filter((l) => !l.includes("ceiling reached")) + .reduce((sum, l) => sum + Buffer.byteLength(l) + 1, 0); + expect(forwardedBytes).toBeLessThanOrEqual(200); + expect(h.lines.filter((l) => l.includes("ceiling reached"))).toHaveLength(1); + }); + + it("force-flushes a newline-free stream instead of buffering it forever", () => { + const h = harness({ maxLineBytes: 16 }); + + h.push("0123456789abcdefghij"); + + expect(h.lines).toEqual(["[engine] 0123456789abcdefghij"]); + }); + + it("ships ceilings that bound a firehose without operator action", () => { + expect(ENGINE_LOG_MAX_TOTAL_BYTES).toBe(32 * 1024 * 1024); + expect(ENGINE_LOG_MAX_BYTES_PER_SECOND).toBe(64 * 1024); + }); +}); From 42bcdce85eacb770ecf45c7315168f8be312d0b4 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 03:01:09 +0800 Subject: [PATCH 19/27] test(cli): pin the rate cap that keeps the death report reachable The forwarder and 8fd1c32's death report share process.stderr, and process.exit(1) discards pending async writes on a pipe. The rate cap is therefore what bounds how much forwarder output can sit ahead of the death report inside any one second. That was measured non-displacing at 64 KiB against a no-forwarder control, but nothing pinned it: the structural guards assert only that the death-detection state, its stderr listener and its exit handler stay outside the forwarder's gate, and all of them still pass if the cap is raised. Assert the ceiling directly, so raising it fails a test instead of silently voiding the measurement. --- test/cli-engine-startup.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/cli-engine-startup.test.ts b/test/cli-engine-startup.test.ts index 32551d7379..82199a6234 100644 --- a/test/cli-engine-startup.test.ts +++ b/test/cli-engine-startup.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; +import { ENGINE_LOG_MAX_BYTES_PER_SECOND } from "../src/cli/engine-log.js"; describe("engine spawn stdio and death detection", () => { const source = readFileSync("src/cli.ts", "utf8"); @@ -47,6 +48,16 @@ describe("engine spawn stdio and death detection", () => { expect(spawnBody.indexOf('child.on("exit"')).toBeGreaterThan(gateEnd); }); + // The forwarder and the death report share process.stderr, and process.exit(1) + // discards pending async writes on a pipe, so the rate cap is what bounds how + // much forwarder output can ever sit ahead of the death report inside one + // second. Measured non-displacing at 64 KiB against a no-forwarder control; + // raising this cap voids that measurement, so it is pinned here rather than + // left to the structural assertions above, which would all still pass. + it("keeps the rate cap low enough not to crowd out the death report", () => { + expect(ENGINE_LOG_MAX_BYTES_PER_SECOND).toBeLessThanOrEqual(64 * 1024); + }); + it("still captures the dying engine's stderr for the death report", () => { expect(spawnBody).toContain( 'const stderr = Buffer.concat(stderrChunks).toString("utf-8")', From b6e0eeffff619d93357cdb5cc10f71b1c9d2cd4f Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 14:10:11 +0800 Subject: [PATCH 20/27] fix(health): evaluate real memory budgets with held severity Compare heap, configured RSS, and visible cgroup budgets independently. Confirm memory-critical entry and recovery over consecutive samples, retain latched failures when telemetry disappears, and render the server evaluation. Signed-off-by: Omar Gerardo --- .env.example | 8 + plugin/skills/agentmemory-config/REFERENCE.md | 10 +- src/health/memory.ts | 204 ++++++++++++++++++ src/health/monitor.ts | 66 ++++-- src/health/thresholds.ts | 41 ++-- src/types.ts | 20 ++ src/viewer/index.html | 54 +++-- test/health-collection.test.ts | 129 +++++++++++ test/health-memory.test.ts | 104 +++++++++ test/health-monitor.test.ts | 62 ++---- test/health-thresholds.test.ts | 175 ++++----------- test/viewer-heap-gauge.test.ts | 104 ++++++--- 12 files changed, 690 insertions(+), 287 deletions(-) create mode 100644 src/health/memory.ts create mode 100644 test/health-collection.test.ts create mode 100644 test/health-memory.test.ts diff --git a/.env.example b/.env.example index 9d346ea191..1e51786d3a 100644 --- a/.env.example +++ b/.env.example @@ -182,3 +182,11 @@ # ----------------------------------------------------------------------------- # OBSIDIAN_AUTO_EXPORT=true # Auto-export memories to an Obsidian vault on every consolidation + +# Health memory pressure: V8, cgroup v2 hard/high limits, and optional RSS budget. +# Committed heap size and the former 512 MiB RSS floor do not define pressure. +# AGENTMEMORY_HEALTH_MEM_WARN_PCT=80 # Require 0 < warn < critical <= 100 +# AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=95 +# AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB=512 # Optional process RSS budget; MB means MiB +# AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES=2 # Positive integer; both entry and recovery, every 30 seconds +# Missing samples break streaks and retain existing critical latches until valid recovery or restart. diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index d12aaed73d..893db4333c 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 37 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 45 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -15,10 +15,17 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_DATA_DIR` - `AGENTMEMORY_DEBUG` - `AGENTMEMORY_DROP_STALE_INDEX` +- `AGENTMEMORY_ENGINE_LOG` +- `AGENTMEMORY_EXIT_ON_ENGINE_DEATH` - `AGENTMEMORY_EXPORT_ROOT` - `AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS` - `AGENTMEMORY_FORCE_PROXY` - `AGENTMEMORY_GRAPH_WEIGHT` +- `AGENTMEMORY_HEALTH_ESCALATE` +- `AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT` +- `AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES` +- `AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB` +- `AGENTMEMORY_HEALTH_MEM_WARN_PCT` - `AGENTMEMORY_III_CONFIG` - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` @@ -26,6 +33,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_INJECT_CONTEXT` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` +- `AGENTMEMORY_MAX_PROVIDER_CONCURRENCY` - `AGENTMEMORY_MCP_BLOCK` - `AGENTMEMORY_PROBE_TIMEOUT_MS` - `AGENTMEMORY_PROJECT_NAME` diff --git a/src/health/memory.ts b/src/health/memory.ts new file mode 100644 index 0000000000..83034a3ef6 --- /dev/null +++ b/src/health/memory.ts @@ -0,0 +1,204 @@ +import { readFile } from "node:fs/promises"; +import { posix } from "node:path"; +import type { CgroupMemory, HealthSnapshot, MemoryEvaluation } from "../types.js"; + +export interface MemoryConfig { + memoryWarnPercent: number; + memoryCriticalPercent: number; + memoryRssBudgetBytes?: number; + memoryHoldSamples: number; +} + +export const MEMORY_DEFAULTS: MemoryConfig = { + memoryWarnPercent: 80, + memoryCriticalPercent: 95, + memoryHoldSamples: 2, +}; + +export function readMemoryConfig( + env: NodeJS.ProcessEnv = process.env, + overrides: Partial = {}, + warn: (message: string) => void = console.warn, +): MemoryConfig { + const number = (key: string, fallback: number) => + env[key] === undefined ? fallback : Number(env[key]); + const config: MemoryConfig = { + memoryWarnPercent: number("AGENTMEMORY_HEALTH_MEM_WARN_PCT", MEMORY_DEFAULTS.memoryWarnPercent), + memoryCriticalPercent: number("AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT", MEMORY_DEFAULTS.memoryCriticalPercent), + memoryHoldSamples: number("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", MEMORY_DEFAULTS.memoryHoldSamples), + ...(env.AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB !== undefined + ? { memoryRssBudgetBytes: Number(env.AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB) * 1024 * 1024 } + : {}), + ...overrides, + }; + if (!(config.memoryWarnPercent > 0 && + config.memoryWarnPercent < config.memoryCriticalPercent && + config.memoryCriticalPercent <= 100)) { + warn("[agentmemory] invalid health memory thresholds: require 0 < warn < critical <= 100; using 80/95"); + config.memoryWarnPercent = MEMORY_DEFAULTS.memoryWarnPercent; + config.memoryCriticalPercent = MEMORY_DEFAULTS.memoryCriticalPercent; + } + if (!Number.isSafeInteger(config.memoryHoldSamples) || config.memoryHoldSamples < 1) { + warn("[agentmemory] invalid health memory hold count: using 2 samples"); + config.memoryHoldSamples = MEMORY_DEFAULTS.memoryHoldSamples; + } + if (config.memoryRssBudgetBytes !== undefined && + !(Number.isFinite(config.memoryRssBudgetBytes) && + config.memoryRssBudgetBytes > 0 && + config.memoryRssBudgetBytes <= Number.MAX_SAFE_INTEGER)) { + warn("[agentmemory] invalid health memory RSS budget: leaving budget unconfigured"); + delete config.memoryRssBudgetBytes; + } + return config; +} + +function parseBytes(raw: string, limit: boolean): number | undefined { + const value = raw.trim(); + if (!/^\d+$/.test(value)) return undefined; + const bytes = Number(value); + return Number.isSafeInteger(bytes) && (limit ? bytes > 0 : bytes >= 0) ? bytes : undefined; +} + +export async function collectCgroupMemory( + read: (path: string) => Promise = path => readFile(path, "utf8"), + platform: string = process.platform, +): Promise { + if (platform !== "linux") return { status: "unsupported", levels: [] }; + try { + const [cgroup, mountinfo] = await Promise.all([ + read("/proc/self/cgroup"), read("/proc/self/mountinfo"), + ]); + const group = cgroup.split("\n").find(line => line.startsWith("0::"))?.slice(3); + const decode = (path: string) => path.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(parseInt(octal, 8))); + const mounts = mountinfo.split("\n").filter(line => line.split(" - ")[1]?.split(" ")[0] === "cgroup2").map(line => { + const fields = line.split(" - ")[0].split(" "); + return { root: decode(fields[3] ?? ""), mount: decode(fields[4] ?? "") }; + }).filter(({ root, mount }) => root.startsWith("/") && mount.startsWith("/")); + if (!group?.startsWith("/") || group.split("/").includes("..") || !mounts.length) { + return { status: "unavailable", levels: [] }; + } + const selected = mounts.sort((a, b) => a.root.length - b.root.length).find(({ root }) => + root === group || group.startsWith(root === "/" ? "/" : root + "/"), + ); + if (!selected) return { status: "unavailable", levels: [] }; + const relative = selected.root !== "/" && + (group === selected.root || group.startsWith(selected.root + "/")) + ? group.slice(selected.root.length) || "/" : group; + const boundary = posix.normalize(selected.mount); + let current = posix.resolve(boundary, "." + relative); + const paths: string[] = []; + while (true) { + paths.push(current); + if (current === boundary) break; + current = posix.dirname(current); + } + let partial = false; + const levels = await Promise.all(paths.map(async path => { + const level: CgroupMemory["levels"][number] = { path: "/" + posix.relative(boundary, path) }; + await Promise.all((["current", "max", "high"] as const).map(async key => { + try { + const raw = await read(posix.join(path, `memory.${key}`)); + if (key !== "current" && raw.trim() === "max") return; + const value = parseBytes(raw, key !== "current"); + if (value === undefined) partial = true; + else level[key] = value; + } catch { partial = true; } + })); + return level; + })); + return { status: partial ? "partial" : "available", levels }; + } catch { return { status: "unavailable", levels: [] }; } +} + +export function evaluateMemory( + memory: HealthSnapshot["memory"], + config: Pick, +): MemoryEvaluation[] { + const evaluations: MemoryEvaluation[] = []; + const add = ( + source: MemoryEvaluation["source"], usedBytes?: number, limitBytes?: number, path?: string, + ) => { + const validUsage = Number.isSafeInteger(usedBytes) && usedBytes! >= 0; + const validLimit = Number.isFinite(limitBytes) && limitBytes! > 0 && limitBytes! <= Number.MAX_SAFE_INTEGER; + const percent = validUsage && validLimit ? usedBytes! / limitBytes! * 100 : undefined; + const severity = percent === undefined ? "healthy" + : source !== "cgroup-high" && percent > config.memoryCriticalPercent ? "critical" + : percent > config.memoryWarnPercent ? "degraded" : "healthy"; + evaluations.push({ + source, + ...(path !== undefined ? { path } : {}), + ...(validUsage ? { usedBytes } : {}), + ...(validLimit ? { limitBytes } : {}), + ...(percent !== undefined ? { percent } : {}), + available: percent !== undefined, + severity, + }); + }; + add("heap", memory.heapUsed, memory.heapSizeLimit); + if (config.memoryRssBudgetBytes !== undefined) add("rss", memory.rss, config.memoryRssBudgetBytes); + for (const level of memory.cgroup?.levels ?? []) { + add("cgroup-max", level.current, level.max, level.path); + add("cgroup-high", level.current, level.high, level.path); + } + return evaluations; +} + +export interface MemoryHoldState { + entry: number; + clear: number; + latched: boolean; + source: MemoryEvaluation["source"]; + path?: string; +} + +export function holdMemoryCritical( + evaluations: MemoryEvaluation[], + states: Map, + requiredSamples: number, +): MemoryEvaluation[] { + const key = (evaluation: MemoryEvaluation) => `${evaluation.source}:${evaluation.path ?? ""}`; + const result = evaluations.map(evaluation => ({ ...evaluation })); + const present = new Set(result.map(key)); + for (const [id, state] of states) { + if (!present.has(id)) result.push({ + source: state.source, path: state.path, + available: false, severity: "healthy", + }); + } + for (const evaluation of result) { + if (evaluation.source === "cgroup-high") continue; + const id = key(evaluation); + const state = states.get(id) ?? { + entry: 0, clear: 0, latched: false, source: evaluation.source, path: evaluation.path, + }; + if (!evaluation.available) { + state.entry = state.clear = 0; + if (state.latched) { + evaluation.severity = "critical"; + evaluation.transition = "unavailable"; + } + } else if (evaluation.severity === "critical") { + state.clear = 0; + state.entry = Math.min(state.entry + 1, requiredSamples); + if (state.entry >= requiredSamples) state.latched = true; + if (!state.latched) { + evaluation.severity = "degraded"; + evaluation.transition = "entering"; + evaluation.samples = state.entry; + } + } else { + state.entry = 0; + state.clear = Math.min(state.clear + 1, requiredSamples); + if (state.clear >= requiredSamples) state.latched = false; + if (state.latched) { + evaluation.severity = "critical"; + evaluation.transition = "recovering"; + evaluation.samples = state.clear; + } + } + if (evaluation.transition) evaluation.requiredSamples = requiredSamples; + if (state.latched || state.entry > 0) states.set(id, state); + else states.delete(id); + } + return result; +} diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 6985be3e4f..f517a7202d 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -4,12 +4,14 @@ import type { HealthSnapshot } from "../types.js"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { evaluateHealth } from "./thresholds.js"; +import { + collectCgroupMemory, evaluateMemory, holdMemoryCritical, readMemoryConfig, + type MemoryConfig, type MemoryHoldState, +} from "./memory.js"; export interface EscalationState { - /** Consecutive snapshots whose KV probe failed. Reset by any healthy probe. */ - consecutiveStalls: number; - /** Set once a healthy KV probe has been seen. Nothing escalates before then. */ - armed: boolean; + consecutiveKvProbeFailures: number; + hasSeenHealthyKvProbe: boolean; escalated: boolean; } @@ -21,7 +23,7 @@ export interface EscalationState { * lag, CPU, memory), so gating on the aggregate would let a CPU spike during a * consolidation pass kill a process that is not wedged at all. * - * `armed` mirrors the shell watchdog's "wait for a first success" rule. Without + * `hasSeenHealthyKvProbe` mirrors the shell watchdog's "wait for a first success" rule. Without * it, a store that is already stalled at boot escalates on the first minute of * every container life, which spends the platform's restart budget in minutes * and ends at a stopped deployment. @@ -38,13 +40,13 @@ export function bumpEscalation( ): boolean { const stalled = snapshot.kvConnectivity?.status === "error"; if (!stalled) { - state.armed = true; - state.consecutiveStalls = 0; + state.hasSeenHealthyKvProbe = true; + state.consecutiveKvProbeFailures = 0; return false; } - if (!state.armed) return false; - state.consecutiveStalls += 1; - if (state.escalated || state.consecutiveStalls < threshold) return false; + if (!state.hasSeenHealthyKvProbe) return false; + state.consecutiveKvProbeFailures += 1; + if (state.escalated || state.consecutiveKvProbeFailures < threshold) return false; state.escalated = true; return true; } @@ -52,10 +54,32 @@ export function bumpEscalation( export function registerHealthMonitor( sdk: ISdk, kv: StateKV, + config: Partial = {}, ): { stop: () => void } { + const memoryConfig = readMemoryConfig(process.env, config); + const memoryStates = new Map(); + let sequence = 0; + let acceptedSequence = 0; + let stopped = false; + let pendingSnapshot: HealthSnapshot | undefined; + let persisting = false; + + async function persistLatest(snapshot: HealthSnapshot): Promise { + pendingSnapshot = snapshot; + if (persisting) return; + persisting = true; + try { + while (pendingSnapshot && !stopped) { + const next = pendingSnapshot; + pendingSnapshot = undefined; + await kv.set(KV.health, "latest", next).catch(() => {}); + } + } finally { persisting = false; } + } + const escalationState: EscalationState = { - consecutiveStalls: 0, - armed: false, + consecutiveKvProbeFailures: 0, + hasSeenHealthyKvProbe: false, escalated: false, }; // Default off. Enabling this arms an automatic process-killer, so it stays @@ -95,7 +119,8 @@ export function registerHealthMonitor( }); } - async function collectHealth(): Promise { + async function collectHealth(): Promise { + const sampleSequence = ++sequence; const mem = process.memoryUsage(); const currentCpu = process.cpuUsage(); const now = Date.now(); @@ -109,6 +134,7 @@ export function registerHealthMonitor( prevCpuUsage = currentCpu; prevCpuTime = now; + const cgroup = await collectCgroupMemory(); const startMark = performance.now(); await new Promise((resolve) => setImmediate(resolve)); const eventLoopLagMs = performance.now() - startMark; @@ -159,6 +185,7 @@ export function registerHealthMonitor( rss: mem.rss, external: mem.external, heapSizeLimit: getHeapStatistics().heap_size_limit, + cgroup, }, cpu: { userMicros: currentCpu.user, @@ -172,7 +199,13 @@ export function registerHealthMonitor( alerts: [], }; - const evaluated = evaluateHealth(snapshot); + if (stopped || sampleSequence <= acceptedSequence) return; + acceptedSequence = sampleSequence; + const memoryEvaluations = holdMemoryCritical( + evaluateMemory(snapshot.memory, memoryConfig), memoryStates, memoryConfig.memoryHoldSamples, + ); + snapshot.memory.evaluations = memoryEvaluations; + const evaluated = evaluateHealth(snapshot, memoryConfig, memoryEvaluations); snapshot.status = evaluated.status; snapshot.alerts = evaluated.alerts; snapshot.notes = evaluated.notes; @@ -182,8 +215,7 @@ export function registerHealthMonitor( escalate(snapshot.alerts); } - await kv.set(KV.health, "latest", snapshot).catch(() => {}); - return snapshot; + await persistLatest(snapshot); } collectHealth().catch(() => {}); @@ -193,7 +225,7 @@ export function registerHealthMonitor( interval.unref(); return { - stop: () => clearInterval(interval), + stop: () => { stopped = true; clearInterval(interval); }, }; } diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index cf5bd11083..0e2b0d2e6a 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -1,13 +1,12 @@ +import { evaluateMemory, MEMORY_DEFAULTS } from "./memory.js"; +import type { MemoryConfig } from "./memory.js"; import type { HealthSnapshot } from "../types.js"; -interface ThresholdConfig { +interface ThresholdConfig extends Omit { eventLoopLagWarnMs: number; eventLoopLagCriticalMs: number; cpuWarnPercent: number; cpuCriticalPercent: number; - memoryWarnPercent: number; - memoryCriticalPercent: number; - memoryRssFloorBytes: number; } const DEFAULTS: ThresholdConfig = { @@ -15,14 +14,13 @@ const DEFAULTS: ThresholdConfig = { eventLoopLagCriticalMs: 500, cpuWarnPercent: 80, cpuCriticalPercent: 90, - memoryWarnPercent: 80, - memoryCriticalPercent: 95, - memoryRssFloorBytes: 512 * 1024 * 1024, + ...MEMORY_DEFAULTS, }; export function evaluateHealth( snapshot: HealthSnapshot, config: Partial = {}, + memoryEvaluations = evaluateMemory(snapshot.memory, { ...DEFAULTS, ...config }), ): { status: "healthy" | "degraded" | "critical"; alerts: string[]; notes: string[] } { const cfg = { ...DEFAULTS, ...config }; const alerts: string[] = []; @@ -77,26 +75,15 @@ export function evaluateHealth( degraded = true; } - // heapTotal is what V8 has committed so far, not what it may grow to, and V8 - // sizes it to demand — so a healthy busy process sits near 100% of it - // permanently. Measure against heap_size_limit when the snapshot carries it. - const heapCeiling = - snapshot.memory.heapSizeLimit && snapshot.memory.heapSizeLimit > 0 - ? snapshot.memory.heapSizeLimit - : snapshot.memory.heapTotal; - const memPercent = - heapCeiling > 0 ? (snapshot.memory.heapUsed / heapCeiling) * 100 : 0; - const rss = snapshot.memory.rss ?? 0; - const rssAboveFloor = rss >= cfg.memoryRssFloorBytes; - const memMb = Math.round(rss / (1024 * 1024)); - if (memPercent > cfg.memoryCriticalPercent && rssAboveFloor) { - alerts.push(`memory_critical_${Math.round(memPercent)}%_rss${memMb}mb`); - critical = true; - } else if (memPercent > cfg.memoryWarnPercent && rssAboveFloor) { - alerts.push(`memory_warn_${Math.round(memPercent)}%_rss${memMb}mb`); - degraded = true; - } else if (memPercent > cfg.memoryWarnPercent) { - notes.push(`memory_heap_tight_${Math.round(memPercent)}%_rss${memMb}mb`); + for (const evaluation of memoryEvaluations) { + const signal = `${evaluation.source}${evaluation.path ?? ""}`; + if (!evaluation.available) notes.push(`memory_unavailable_${signal}`); + if (evaluation.transition) notes.push(`memory_${evaluation.transition}_${signal}`); + if (evaluation.severity === "critical") critical = true; + if (evaluation.severity === "degraded") degraded = true; + if (evaluation.severity !== "healthy") { + alerts.push(`memory_${evaluation.severity === "critical" ? "critical" : "warn"}_${signal}_${evaluation.percent === undefined ? "unavailable" : `${Math.round(evaluation.percent)}%`}`); + } } const status = critical ? "critical" : degraded ? "degraded" : "healthy"; diff --git a/src/types.ts b/src/types.ts index d8543c13ab..5823d11905 100644 --- a/src/types.ts +++ b/src/types.ts @@ -218,6 +218,24 @@ export interface FunctionMetrics { avgQualityScore: number; } +export interface MemoryEvaluation { + source: "heap" | "rss" | "cgroup-max" | "cgroup-high"; + path?: string; + usedBytes?: number; + limitBytes?: number; + percent?: number; + severity: "healthy" | "degraded" | "critical"; + available: boolean; + transition?: "entering" | "recovering" | "unavailable"; + samples?: number; + requiredSamples?: number; +} + +export interface CgroupMemory { + status: "available" | "partial" | "unavailable" | "unsupported"; + levels: Array<{ path: string; current?: number; max?: number; high?: number }>; +} + export interface HealthSnapshot { connectionState: string; workers: Array<{ id: string; name: string; status: string }>; @@ -227,6 +245,8 @@ export interface HealthSnapshot { rss: number; external: number; heapSizeLimit?: number; + cgroup?: CgroupMemory; + evaluations?: MemoryEvaluation[]; }; cpu: { userMicros: number; systemMicros: number; percent: number }; eventLoopLagMs: number; diff --git a/src/viewer/index.html b/src/viewer/index.html index c532a76e9b..85b58e7e37 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -870,6 +870,11 @@ .gauge-bar { flex: 1; height: 6px; background: var(--bg-inset); overflow: hidden; } .gauge-fill { height: 100%; transition: width 0.5s; } .gauge-label { width: 90px; font-size: 10px; color: var(--ink-muted); font-family: var(--font-ui); text-transform: uppercase; letter-spacing: 0.08em; font-weight: 600; } + .memory-gauge .gauge { flex-wrap: wrap; } + .memory-gauge .gauge-label { width: 130px; } + .memory-gauge .gauge-value { width: auto; min-width: 155px; } + .memory-detail { font-size: 11px; color: var(--ink-muted); overflow-wrap: anywhere; margin: 4px 0 8px; } + @media (max-width: 600px) { .memory-gauge .gauge-value { width: 100%; } } .gauge-value { width: 70px; font-size: 11px; color: var(--ink); text-align: right; font-family: var(--font-mono); } .obs-type-icon { font-size: 16px; margin-right: 4px; } @@ -1202,6 +1207,11 @@

agentmemory

// into sentences; unknown slugs render as-is. function humanizeHealthFlag(f) { var m; + var memoryLabels = { heap: 'V8 heap', rss: 'RSS budget', 'cgroup-max': 'Container limit', 'cgroup-high': 'Container throttle' }; + if ((m = /^memory_(warn|critical)_(heap|rss|cgroup-max|cgroup-high)(.*?)_(\d+%|unavailable)$/.exec(f))) + return memoryLabels[m[2]] + (m[3] ? ' (' + m[3] + ')' : '') + ': ' + (m[1] === 'critical' ? 'critical' : 'elevated') + ', ' + (m[4] === 'unavailable' ? 'measurement unavailable' : m[4] + ' of budget') + '.'; + if ((m = /^memory_(entering|recovering|unavailable)_(heap|rss|cgroup-max|cgroup-high)(.*)$/.exec(f))) + return memoryLabels[m[2]] + (m[3] ? ' (' + m[3] + ')' : '') + ': ' + ({ entering: 'Critical pending', recovering: 'Recovery pending', unavailable: 'Measurement unavailable' })[m[1]] + '.'; if ((m = /^memory_heap_tight_(\d+)%_rss(\d+)mb$/.exec(f))) return 'Heap is running tight: ' + m[1] + '% of allocated heap in use (process memory ' + m[2] + ' MB). Informational — Node grows the heap on demand.'; if ((m = /^memory_(warn|critical)_(\d+)%_rss(\d+)mb$/.exec(f))) @@ -1467,6 +1477,29 @@

agentmemory

} } + function renderMemoryResources(memory) { + var labels = { heap: 'Heap', rss: 'RSS budget', 'cgroup-max': 'Container limit', 'cgroup-high': 'Container throttle' }; + var colors = { healthy: 'var(--green)', degraded: 'var(--yellow)', critical: 'var(--red)' }; + var transitions = { entering: 'Critical pending', recovering: 'Recovery pending', unavailable: 'Measurement unavailable' }; + var html = '
'; + var evaluations = Array.isArray(memory.evaluations) ? memory.evaluations : []; + evaluations.forEach(function(item) { + var available = item.available && Number.isFinite(item.percent) && item.percent >= 0 && Number.isFinite(item.usedBytes) && Number.isFinite(item.limitBytes) && item.limitBytes > 0; + var percent = available ? item.percent : 0; + var color = colors[item.severity] || 'var(--ink-muted)'; + var value = available ? Math.round(item.usedBytes / 1024 / 1024) + ' / ' + Math.round(item.limitBytes / 1024 / 1024) + ' MiB · ' + percent.toFixed(1) + '%' : 'Unavailable'; + html += '
' + (labels[item.source] || 'Memory') + '
' + value + '
'; + var detail = transitions[item.transition] || ''; + if (item.path) detail += (detail ? ' · ' : '') + item.path; + if (detail) html += '
' + esc(detail) + '
'; + }); + if (!evaluations.length) html += '
Memory pressure unavailable
'; + if (memory.cgroup && memory.cgroup.status !== 'available') html += '
Container measurements: ' + esc(memory.cgroup.status) + '
'; + html += '
Heap: ' + Math.round((memory.heapUsed || 0) / 1024 / 1024) + ' MiB · RSS: ' + Math.round((memory.rss || 0) / 1024 / 1024) + ' MiB'; + if (memory.external) html += ' · External: ' + Math.round(memory.external / 1024 / 1024) + ' MiB'; + return html + '
'; + } + async function loadDashboard() { var el = document.getElementById('view-dashboard'); if (!state.dashboard.loaded) el.innerHTML = '
Loading dashboard...
'; @@ -1575,26 +1608,7 @@

agentmemory

if (snap.memory || snap.cpu) { html += '
System Resources
'; - if (snap.memory) { - var heapUsed = Math.round((snap.memory.heapUsed || 0) / 1024 / 1024); - var heapTotal = Math.round((snap.memory.heapTotal || 0) / 1024 / 1024); - var rss = Math.round((snap.memory.rss || 0) / 1024 / 1024); - var limitBytes = snap.memory.heapSizeLimit || 0; - var ceilingBytes = limitBytes > 0 ? limitBytes : (snap.memory.heapTotal || 0); - var heapCeiling = Math.round(ceilingBytes / 1024 / 1024); - // Percentage off raw bytes so the gauge agrees with evaluateHealth, - // which never sees the MB rounding used for the label. - var heapPercent = ceilingBytes > 0 ? ((snap.memory.heapUsed || 0) / ceilingBytes) * 100 : 0; - var heapPct = Math.round(heapPercent); - var rssAboveFloor = rss >= 512; - var heapColor = (heapPercent > 80 && rssAboveFloor) ? 'var(--red)' : (heapPercent > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; - html += '
Heap
' + heapUsed + ' / ' + heapCeiling + ' MB
'; - html += '
RSS
' + rss + ' MB
'; - if (snap.memory.external) { - var ext = Math.round(snap.memory.external / 1024 / 1024); - html += '
External
' + ext + ' MB
'; - } - } + if (snap.memory) html += renderMemoryResources(snap.memory); if (snap.cpu) { var cpuPct = snap.cpu.percent || 0; var cpuColor = cpuPct > 80 ? 'var(--red)' : cpuPct > 50 ? 'var(--yellow)' : 'var(--green)'; diff --git a/test/health-collection.test.ts b/test/health-collection.test.ts new file mode 100644 index 0000000000..843042d583 --- /dev/null +++ b/test/health-collection.test.ts @@ -0,0 +1,129 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ISdk } from "iii-sdk"; +import type { StateKV } from "../src/state/kv.js"; +import type { HealthSnapshot } from "../src/types.js"; +import { KV } from "../src/state/schema.js"; +vi.mock("node:v8", () => ({ getHeapStatistics: vi.fn(() => ({ heap_size_limit: 100 })) })); +vi.mock("../src/health/memory.js", async importOriginal => ({ ...await importOriginal(), collectCgroupMemory: vi.fn(async () => ({ status: "unsupported", levels: [] })) })); +import { collectCgroupMemory } from "../src/health/memory.js"; +import { registerHealthMonitor } from "../src/health/monitor.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; + +let used = 96; +let stop: (() => void) | undefined; +beforeEach(() => { + vi.useFakeTimers(); + used = 96; + vi.spyOn(process, "memoryUsage").mockImplementation(() => ({ heapUsed: used, heapTotal: 100, rss: 200, external: 0, arrayBuffers: 0 })); + vi.spyOn(process, "cpuUsage").mockReturnValue({ user: 0, system: 0 }); + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "unsupported", levels: [] }); +}); +afterEach(() => { stop?.(); vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); }); + +function setup() { + const stored = new Map(); + const history: HealthSnapshot[] = []; + const functions = new Map Promise>(); + const sdk = { on: vi.fn(), trigger: vi.fn(async () => ({ workers: [] })), registerFunction: (id: string, fn: (payload: unknown) => Promise) => functions.set(id, fn), registerTrigger: vi.fn() }; + const kv = { set: vi.fn(async (_scope: string, key: string, value: unknown) => { stored.set(key, value); if (key === "latest") history.push(value as HealthSnapshot); }), get: vi.fn(async (_scope: string, key: string) => stored.get(key)), list: vi.fn(async () => []) }; + registerApiTriggers(sdk as unknown as ISdk, kv as unknown as StateKV); + stop = registerHealthMonitor(sdk as unknown as ISdk, kv as unknown as StateKV).stop; + return { sdk, kv, history, health: () => functions.get("api::health")!({}), latest: () => stored.get("latest") as HealthSnapshot }; +} +async function flush() { await vi.advanceTimersByTimeAsync(1); } +async function tick() { await vi.advanceTimersByTimeAsync(30_000); } + +describe("periodic memory hold and persisted API health", () => { + it("requires two observations to enter and clear; polls cannot advance the hold", async () => { + const app = setup(); await flush(); + expect(app.latest().status).toBe("degraded"); + expect(app.latest().memory.evaluations?.[0].transition).toBe("entering"); + for (let i = 0; i < 3; i++) expect((await app.health()).status_code).toBe(200); + await tick(); expect((await app.health()).status_code).toBe(503); + used = 20; await tick(); + expect(app.latest().memory.evaluations?.[0].transition).toBe("recovering"); + expect((await app.health()).status_code).toBe(503); + await tick(); expect((await app.health()).status_code).toBe(200); + }); + + it("interrupts spikes, missing entry samples and recovery samples", async () => { + const app = setup(); await flush(); + used = 20; await tick(); used = 96; await tick(); expect(app.latest().status).toBe("degraded"); + used = NaN; await tick(); used = 96; await tick(); expect(app.latest().status).toBe("degraded"); + await tick(); expect(app.latest().status).toBe("critical"); + used = 20; await tick(); used = NaN; await tick(); + expect(app.latest().memory.evaluations?.[0]).toMatchObject({ severity: "critical", available: false, transition: "unavailable" }); + used = 20; await tick(); expect(app.latest().status).toBe("critical"); + await tick(); expect(app.latest().status).toBe("healthy"); + }); + + it("retains a vanished critical cgroup source and resets on monitor restart", async () => { + used = 1; + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "available", levels: [{ path: "/parent", current: 96, max: 100 }] }); + const app = setup(); await flush(); await tick(); expect(app.latest().status).toBe("critical"); + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "unavailable", levels: [] }); + await tick(); expect(app.latest().memory.evaluations).toContainEqual(expect.objectContaining({ path: "/parent", severity: "critical", transition: "unavailable" })); + stop?.(); stop = registerHealthMonitor(app.sdk as unknown as ISdk, app.kv as unknown as StateKV).stop; + await flush(); expect(app.latest().status).toBe("healthy"); + }); + + it("reads configuration once and keeps KV failures immediate", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", "3"); + const app = setup(); await flush(); + vi.stubEnv("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", "1"); + await tick(); expect(app.latest().status).toBe("degraded"); + app.kv.get.mockRejectedValue(new Error("KV unavailable")); + used = 20; await tick(); + expect(app.latest().status).toBe("critical"); + expect(app.latest().alerts).toContain("kv_probe_failed"); + }); + + it("discards out-of-order probe completions without double-counting entry", async () => { + let release: (value: { status: "unsupported"; levels: [] }) => void; + vi.mocked(collectCgroupMemory).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + const app = setup(); await flush(); await tick(); + expect(app.latest().status).toBe("degraded"); + release!({ status: "unsupported", levels: [] }); await flush(); + expect(app.history).toHaveLength(1); + await tick(); expect(app.latest().status).toBe("critical"); + }); + + it("escalates ten failed probes despite a stalled persist and times out workers", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_ESCALATE", "true"); + const kill = vi.spyOn(process, "kill").mockReturnValue(true); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const app = setup(); await flush(); + const original = app.kv.set.getMockImplementation()!; + app.kv.set.mockImplementation((scope, key, value) => key === "latest" ? new Promise(() => {}) : original(scope, key, value)); + app.kv.get.mockRejectedValue(new Error("store unavailable")); + app.sdk.trigger.mockImplementation(() => new Promise(() => {})); + for (let i = 0; i < 9; i++) await tick(); + await vi.advanceTimersByTimeAsync(5000); + expect(kill).not.toHaveBeenCalled(); + await tick(); + expect(kill).toHaveBeenCalledExactlyOnceWith(process.pid, "SIGTERM"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("KV unreachable")); + }); + + it("serializes slow persists while probes and escalation keep running", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_ESCALATE", "true"); + const kill = vi.spyOn(process, "kill").mockReturnValue(true); + const app = setup(); await flush(); + const original = app.kv.set.getMockImplementation()!; + let release: () => void; + let active = 0; + let maximum = 0; + app.kv.set.mockImplementation(async (scope, key, value) => { + if (key === "latest") { + active++; maximum = Math.max(maximum, active); + if (!release) await new Promise(resolve => { release = resolve; }); + await original(scope, key, value); active--; + } else await original(scope, key, value); + }); + await tick(); used = 20; await tick(); await tick(); + expect(maximum).toBe(1); + release!(); await flush(); expect(app.latest().status).toBe("healthy"); + expect(kill).not.toHaveBeenCalled(); + expect(app.kv.set).toHaveBeenCalledWith(KV.health, "_probe", expect.anything()); + }); +}); diff --git a/test/health-memory.test.ts b/test/health-memory.test.ts new file mode 100644 index 0000000000..ebb6b97f32 --- /dev/null +++ b/test/health-memory.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it, vi } from "vitest"; +import { collectCgroupMemory, readMemoryConfig } from "../src/health/memory.js"; + +const mount = "29 23 0:26 / /sys/fs/cgroup rw,nosuid - cgroup2 cgroup rw"; +function reader(files: Record) { + return async (path: string) => { + if (!(path in files)) throw new Error("unreadable"); + return files[path]; + }; +} +function files(group = "/service") { + return { "/proc/self/cgroup": `0::${group}\n`, "/proc/self/mountinfo": mount }; +} + +describe("cgroup v2 measurement", () => { + it("pairs each visible ancestor's usage with its own refreshed limits", async () => { + const data = { ...files(), "/sys/fs/cgroup/service/memory.current": "10", "/sys/fs/cgroup/service/memory.max": "1000", "/sys/fs/cgroup/service/memory.high": "max", "/sys/fs/cgroup/memory.current": "99", "/sys/fs/cgroup/memory.max": "100", "/sys/fs/cgroup/memory.high": "80" }; + expect(await collectCgroupMemory(reader(data), "linux")).toEqual({ status: "available", levels: [ { path: "/service", current: 10, max: 1000 }, { path: "/", current: 99, max: 100, high: 80 } ] }); + data["/sys/fs/cgroup/memory.max"] = "200"; + expect((await collectCgroupMemory(reader(data), "linux")).levels[1].max).toBe(200); + }); + + it.each([ + ["/tenant", "/tenant/service", "/service"], + ["/", "/", "/"], + ["/", "/service", "/service"], + ])("resolves mount root %s and process path %s", async (root, group, expected) => { + const result = await collectCgroupMemory(reader({ ...files(group), "/proc/self/mountinfo": mount.replace(" / /sys", ` ${root} /sys`), [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.current`]: "10", [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.max`]: "100", [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.high`]: "max" }), "linux"); + expect(result.levels[0]).toMatchObject({ path: expected, current: 10, max: 100 }); + }); + + it.each(["/other", "/.."])("does not attribute an unmatched mount root %s to this process", async root => { + const read = vi.fn(reader({ + ...files("/app"), + "/proc/self/mountinfo": mount.replace(" / /sys", ` ${root} /sys`), + "/sys/fs/cgroup/app/memory.current": "99", + "/sys/fs/cgroup/app/memory.max": "100", + })); + expect(await collectCgroupMemory(read, "linux")).toEqual({ status: "unavailable", levels: [] }); + expect(read.mock.calls.every(([path]) => path.startsWith("/proc/"))).toBe(true); + }); + + it("uses the mount exposing the full visible ancestry", async () => { + const result = await collectCgroupMemory(reader({ + ...files("/tenant/service"), + "/proc/self/mountinfo": mount.replace(" / /sys/fs/cgroup", " /tenant /tenant-cgroup") + "\n" + mount, + "/sys/fs/cgroup/tenant/service/memory.current": "1", + "/sys/fs/cgroup/tenant/service/memory.max": "100", + "/sys/fs/cgroup/tenant/memory.current": "10", + "/sys/fs/cgroup/tenant/memory.max": "100", + "/sys/fs/cgroup/memory.current": "99", + "/sys/fs/cgroup/memory.max": "100", + }), "linux"); + expect(result.levels).toContainEqual({ path: "/", current: 99, max: 100 }); + }); + + it("decodes escaped mount paths and handles high independently", async () => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/proc/self/mountinfo": mount.replace("/sys/fs/cgroup", "/cg\\040space"), "/cg space/memory.current": "90", "/cg space/memory.high": "80" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", current: 90, high: 80 }] }); + }); + + it.each(["", "-1", "NaN", "Infinity", "1e3", "12oops", "9007199254740992"])("rejects malformed usage %s without discarding valid limits", async current => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": current, "/sys/fs/cgroup/memory.max": "100", "/sys/fs/cgroup/memory.high": "max" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", max: 100 }] }); + }); + + it.each(["0", "-1", "", "Infinity", "1024kb", "9007199254740992"])("rejects invalid hard limit %s while retaining high", async max => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": "90", "/sys/fs/cgroup/memory.max": max, "/sys/fs/cgroup/memory.high": "80" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", current: 90, high: 80 }] }); + }); + + it("accepts zero usage and unlimited limits", async () => { + expect(await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": "0\n", "/sys/fs/cgroup/memory.max": "max", "/sys/fs/cgroup/memory.high": "max" }), "linux")).toEqual({ status: "available", levels: [{ path: "/", current: 0 }] }); + }); + + it("handles unavailable and unsupported hosts", async () => { + expect((await collectCgroupMemory(reader({}), "linux")).status).toBe("unavailable"); + const read = vi.fn(); + expect((await collectCgroupMemory(read, "darwin")).status).toBe("unsupported"); + expect(read).not.toHaveBeenCalled(); + }); +}); + +describe("memory configuration", () => { + it("uses defaults and explicit overrides", () => { + expect(readMemoryConfig({}, {}, vi.fn())).toEqual({ memoryWarnPercent: 80, memoryCriticalPercent: 95, memoryHoldSamples: 2 }); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: "70", AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "90", AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB: "128", AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES: "3" }, { memoryHoldSamples: 1 }, vi.fn())).toEqual({ memoryWarnPercent: 70, memoryCriticalPercent: 90, memoryRssBudgetBytes: 128 * 1024 * 1024, memoryHoldSamples: 1 }); + }); + it.each(["", "0", "-1", "NaN", "Infinity", "101", "95"])("rejects invalid warning %s", value => { + const warn = vi.fn(); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: value }, {}, warn).memoryWarnPercent).toBe(80); + expect(warn).toHaveBeenCalledTimes(1); + }); + it("accepts critical 100 and rejects crossed thresholds", () => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "100" }, {}, vi.fn()).memoryCriticalPercent).toBe(100); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: "90", AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "80" }, {}, vi.fn())).toMatchObject({ memoryWarnPercent: 80, memoryCriticalPercent: 95 }); + }); + it.each(["0", "-1", "1.5", "NaN", "Infinity"])("rejects hold count %s", value => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES: value }, {}, vi.fn()).memoryHoldSamples).toBe(2); + }); + it.each(["", "0", "-1", "NaN", "Infinity", "1e300"])("rejects budget %s", value => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB: value }, {}, vi.fn()).memoryRssBudgetBytes).toBeUndefined(); + }); +}); diff --git a/test/health-monitor.test.ts b/test/health-monitor.test.ts index 14b42e0f6c..54bbc8bbfb 100644 --- a/test/health-monitor.test.ts +++ b/test/health-monitor.test.ts @@ -1,6 +1,4 @@ import { describe, it, expect } from "vitest"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; import type { HealthSnapshot } from "../src/types.js"; import { bumpEscalation, type EscalationState } from "../src/health/monitor.js"; @@ -22,11 +20,11 @@ function snap(over: Partial = {}): HealthSnapshot { const stalled = () => snap({ kvConnectivity: { status: "error", error: "kv_probe_failed" } }); function fresh(): EscalationState { - return { consecutiveStalls: 0, armed: false, escalated: false }; + return { consecutiveKvProbeFailures: 0, hasSeenHealthyKvProbe: false, escalated: false }; } /** Arm the state the way a healthy first collection would. */ -function armed(): EscalationState { +function hasSeenHealthyKvProbe(): EscalationState { const s = fresh(); bumpEscalation(snap(), s, 3); return s; @@ -41,36 +39,36 @@ describe("bumpEscalation arming", () => { for (let i = 0; i < 50; i++) { expect(bumpEscalation(stalled(), state, 3)).toBe(false); } - expect(state.armed).toBe(false); - expect(state.consecutiveStalls).toBe(0); + expect(state.hasSeenHealthyKvProbe).toBe(false); + expect(state.consecutiveKvProbeFailures).toBe(0); }); it("arms on the first healthy probe", () => { const state = fresh(); bumpEscalation(snap(), state, 3); - expect(state.armed).toBe(true); + expect(state.hasSeenHealthyKvProbe).toBe(true); }); }); describe("bumpEscalation gating", () => { it("escalates on the Nth consecutive stall, not before", () => { - const state = armed(); + const state = hasSeenHealthyKvProbe(); expect(bumpEscalation(stalled(), state, 3)).toBe(false); expect(bumpEscalation(stalled(), state, 3)).toBe(false); expect(bumpEscalation(stalled(), state, 3)).toBe(true); }); it("resets the counter on a healthy probe", () => { - const state = armed(); + const state = hasSeenHealthyKvProbe(); bumpEscalation(stalled(), state, 3); bumpEscalation(stalled(), state, 3); bumpEscalation(snap(), state, 3); - expect(state.consecutiveStalls).toBe(0); + expect(state.consecutiveKvProbeFailures).toBe(0); expect(bumpEscalation(stalled(), state, 3)).toBe(false); }); it("escalates at most once per process", () => { - const state = armed(); + const state = hasSeenHealthyKvProbe(); let fired = 0; for (let i = 0; i < 20; i++) { if (bumpEscalation(stalled(), state, 1)) fired++; @@ -87,56 +85,22 @@ describe("bumpEscalation gating", () => { ["connection", { connectionState: "disconnected" }], ["memory", { memory: { heapUsed: 99, heapTotal: 100, rss: 99, external: 0 } }], ])("never escalates on a %s critical while the KV probe is healthy", (_label, over) => { - const state = armed(); + const state = hasSeenHealthyKvProbe(); for (let i = 0; i < 50; i++) { expect( bumpEscalation(snap({ ...over, status: "critical" } as Partial), state, 3), ).toBe(false); } - expect(state.consecutiveStalls).toBe(0); + expect(state.consecutiveKvProbeFailures).toBe(0); }); it("treats an absent kvConnectivity as no signal, not as a stall", () => { - const state = armed(); + const state = hasSeenHealthyKvProbe(); const s = snap(); delete s.kvConnectivity; for (let i = 0; i < 50; i++) { expect(bumpEscalation(s, state, 3)).toBe(false); } - expect(state.consecutiveStalls).toBe(0); - }); -}); - -// KTD7. The escalation decision must precede the snapshot persist. `kv.set` -// routes through `sdk.trigger`, whose invocationTimeoutMs is 180000 -// (src/index.ts), so a stalled store parks the persist for three minutes. A -// counter placed behind it would never advance during the exact failure it -// exists to catch. -// -// Asserted on source order because no test constructs registerHealthMonitor, so -// the call site is otherwise executed by nothing. Verified discriminating: this -// passes on the real tree and fails on a tree with the two lines swapped. -// Matches the precedent in test/cli-second-instance-guard.test.ts. -describe("collectHealth statement order", () => { - const src = readFileSync( - fileURLToPath(new URL("../src/health/monitor.ts", import.meta.url)), - "utf8", - ); - - it("decides escalation before the un-raced snapshot persist", () => { - const decide = src.indexOf("bumpEscalation(snapshot, escalationState"); - const persist = src.indexOf('kv.set(KV.health, "latest"'); - expect(decide).toBeGreaterThan(-1); - expect(persist).toBeGreaterThan(-1); - expect(decide).toBeLessThan(persist); - }); - - // The workers probe sits upstream of both. Unraced it inherits the engine's - // 180s invocation timeout and delays detection by tens of minutes. - it("races the workers probe so it cannot park the collection", () => { - const workers = src.indexOf('function_id: "engine::workers::list"'); - expect(workers).toBeGreaterThan(-1); - const before = src.slice(Math.max(0, workers - 400), workers); - expect(before).toContain("Promise.race"); + expect(state.consecutiveKvProbeFailures).toBe(0); }); }); diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index bae18455ef..c30f0e652e 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -17,159 +17,56 @@ function snap(over: Partial = {}): HealthSnapshot { }; } -describe("evaluateHealth memory severity", () => { - it("stays healthy when heap fills a tiny steady-state process (issue #158)", () => { - const s = snap({ - memory: { - heapUsed: 45 * 1024 * 1024, - heapTotal: 46 * 1024 * 1024, - rss: 120 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts, notes } = evaluateHealth(s); - expect(status).toBe("healthy"); - expect(alerts.find((a) => a.startsWith("memory_critical_"))).toBeUndefined(); - expect(alerts.find((a) => a.startsWith("memory_warn_"))).toBeUndefined(); - expect(alerts.find((a) => a.startsWith("memory_heap_tight_"))).toBeUndefined(); - expect(notes.find((n) => n.startsWith("memory_heap_tight_"))).toBeDefined(); +describe("independent real memory budgets", () => { + it.each([ + [390_771_624, 404_930_560, 6_492_782_592], + [267_970_328, 277_663_744, 2_197_815_296], + ])("keeps the reported committed heap %s healthy", (heapUsed, heapTotal, heapSizeLimit) => { + expect(evaluateHealth(snap({ memory: { heapUsed, heapTotal, heapSizeLimit, rss: 600_000_000, external: 0 } })).status).toBe("healthy"); }); - it("goes critical when heap ratio is high AND RSS is above the floor", () => { - const s = snap({ - memory: { - heapUsed: 970 * 1024 * 1024, - heapTotal: 1000 * 1024 * 1024, - rss: 1100 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts } = evaluateHealth(s); - expect(status).toBe("critical"); - expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(true); + it("does not invent a budget for old snapshots", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 970, heapTotal: 1000, rss: 900_000_000, external: 0 } })); + expect(result.status).toBe("healthy"); + expect(result.notes).toContain("memory_unavailable_heap"); }); - it("records heap_tight in the warn band when RSS is below the floor", () => { - const s = snap({ - memory: { - heapUsed: 85 * 1024 * 1024, - heapTotal: 100 * 1024 * 1024, - rss: 50 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts, notes } = evaluateHealth(s); - expect(status).toBe("healthy"); - expect(notes.some((n) => n.startsWith("memory_heap_tight_"))).toBe(true); - expect(alerts.some((a) => a.startsWith("memory_heap_tight_"))).toBe(false); - expect(alerts.some((a) => a.startsWith("memory_warn_"))).toBe(false); - expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(false); + it.each([[80, "healthy"], [81, "degraded"], [95, "degraded"], [96, "critical"]])("evaluates small heaps at %s percent without an RSS floor", (heapUsed, expected) => { + expect(evaluateHealth(snap({ memory: { heapUsed: Number(heapUsed), heapTotal: 100, heapSizeLimit: 100, rss: 200, external: 0 } })).status).toBe(expected); }); - it("goes degraded when heap is above warn AND RSS is above the floor", () => { - const s = snap({ - memory: { - heapUsed: 850 * 1024 * 1024, - heapTotal: 1000 * 1024 * 1024, - rss: 900 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts } = evaluateHealth(s, { memoryRssFloorBytes: 800 * 1024 * 1024 }); - expect(status).toBe("degraded"); - expect(alerts.some((a) => a.startsWith("memory_warn_"))).toBe(true); + it("keeps heap and cgroup signals independent", () => { + const memory = { heapUsed: 96, heapTotal: 100, heapSizeLimit: 100, rss: 200, external: 0, cgroup: { status: "available" as const, levels: [{ path: "/", current: 100, max: 1000 }] } }; + expect(evaluateHealth(snap({ memory })).status).toBe("critical"); + memory.heapUsed = 10; + memory.cgroup.levels[0].current = 960; + expect(evaluateHealth(snap({ memory })).status).toBe("critical"); }); - it("respects caller-supplied memoryRssFloorBytes", () => { - const s = snap({ - memory: { - heapUsed: 98, - heapTotal: 100, - rss: 50 * 1024 * 1024, - external: 0, - }, - }); - const loose = evaluateHealth(s, { memoryRssFloorBytes: 10 * 1024 * 1024 }); - expect(loose.status).toBe("critical"); - const strict = evaluateHealth(s, { memoryRssFloorBytes: 1024 * 1024 * 1024 }); - expect(strict.status).toBe("healthy"); + it("reports memory.high as warning even beyond its boundary", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 1, heapTotal: 100, external: 0, rss: 2, cgroup: { status: "available", levels: [{ path: "/", current: 110, high: 100 }] } } })); + expect(result.status).toBe("degraded"); + expect(result.alerts).toContain("memory_warn_cgroup-high/_110%"); }); -}); -describe("evaluateHealth memory severity — denominator", () => { - const LIMIT_6192MB = 6192 * 1024 * 1024; - - it("stays healthy when a busy process fills its committed heap but sits far below the V8 limit", () => { - // Captured from a live deployment: this process reported - // memory_critical_97% while using 6% of the heap it may grow to. - const s = snap({ - memory: { - heapUsed: 390_771_624, - heapTotal: 404_930_560, - rss: 584_327_168, - external: 6_566_392, - heapSizeLimit: LIMIT_6192MB, - }, - }); - - const { status, alerts } = evaluateHealth(s); - - expect(status).toBe("healthy"); - expect(alerts.find((a) => a.startsWith("memory_critical_"))).toBeUndefined(); - expect(alerts.find((a) => a.startsWith("memory_warn_"))).toBeUndefined(); + it("evaluates an explicit RSS budget alongside low cgroup occupancy", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 1, heapTotal: 100, external: 0, rss: 110, cgroup: { status: "available", levels: [{ path: "/", current: 110, max: 1000 }] } } }), { memoryRssBudgetBytes: 100 }); + expect(result.status).toBe("critical"); + expect(result.alerts).toContain("memory_critical_rss_110%"); }); - it("reports the percentage against the limit, not against the committed heap", () => { - const s = snap({ - memory: { - heapUsed: 390_771_624, - heapTotal: 404_930_560, - rss: 584_327_168, - external: 0, - heapSizeLimit: LIMIT_6192MB, - }, - }); - - // Widen the band so the alert fires as a warn either way and the rendered - // percentage is the only thing under test. - const { alerts } = evaluateHealth(s, { - memoryWarnPercent: 1, - memoryCriticalPercent: 99, - }); - const warn = alerts.find((a) => a.startsWith("memory_warn_")); - - // 390771624 / 6492782592 = 6%, not 390771624 / 404930560 = 97%. - expect(warn).toBe("memory_warn_6%_rss557mb"); + it.each([0, -1, Infinity, NaN])("rejects invalid V8 limit %s", heapSizeLimit => { + expect(evaluateHealth(snap({ memory: { heapUsed: 99, heapTotal: 100, heapSizeLimit, rss: 1_000_000_000, external: 0 } })).status).toBe("healthy"); }); +}); - it("still goes critical when the heap genuinely approaches the V8 limit", () => { - const s = snap({ - memory: { - heapUsed: 6_200_000_000, - heapTotal: 6_300_000_000, - rss: 6_800_000_000, - external: 0, - heapSizeLimit: LIMIT_6192MB, - }, - }); - - const { status, alerts } = evaluateHealth(s); - - expect(status).toBe("critical"); - expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(true); - }); - - it("falls back to the committed heap when heapSizeLimit is absent", () => { - const s = snap({ - memory: { - heapUsed: 970 * 1024 * 1024, - heapTotal: 1000 * 1024 * 1024, - rss: 1100 * 1024 * 1024, - external: 0, - }, - }); - - expect(evaluateHealth(s).status).toBe("critical"); +describe("non-memory signals", () => { + it.each([ + { cpu: { userMicros: 0, systemMicros: 0, percent: 91 } }, + { eventLoopLagMs: 501 }, + { connectionState: "failed" }, + ])("keeps critical conditions immediate during memory entry: %s", other => { + expect(evaluateHealth(snap(other), {}, [{ source: "heap", available: true, percent: 99, severity: "degraded", transition: "entering" }]).status).toBe("critical"); }); }); diff --git a/test/viewer-heap-gauge.test.ts b/test/viewer-heap-gauge.test.ts index b99ee4907a..f07247e3ae 100644 --- a/test/viewer-heap-gauge.test.ts +++ b/test/viewer-heap-gauge.test.ts @@ -1,44 +1,80 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; -// The dashboard used to recompute heapUsed / heapTotal client-side, which is -// the same over-reporting the health thresholds had: heapTotal is what V8 has -// committed, not what it may grow to. The gauge now measures against -// heapSizeLimit, and it divides raw byte values so it cannot disagree with -// evaluateHealth over a rounding step. -// -// Asserting on the emitted source rather than running the gauge follows -// viewer-graph-cooldown and viewer-memories-sort: the viewer ships as one -// HTML file with inline JS, so there is no module to import and execute. -describe("viewer heap gauge", () => { - const viewer = readFileSync("src/viewer/index.html", "utf-8"); - - it("measures against the V8 heap limit, falling back to heapTotal", () => { - expect(viewer).toMatch(/limitBytes\s*=\s*snap\.memory\.heapSizeLimit\s*\|\|\s*0/); - expect(viewer).toMatch( - /ceilingBytes\s*=\s*limitBytes\s*>\s*0\s*\?\s*limitBytes\s*:\s*\(snap\.memory\.heapTotal\s*\|\|\s*0\)/, - ); +const viewer = readFileSync("src/viewer/index.html", "utf8"); +function render(memory: Record): string { + const start = viewer.indexOf(" function renderMemoryResources(memory) {"); + const end = viewer.indexOf("\n }", start) + 6; + expect(start).toBeGreaterThan(-1); + return runInNewContext(`${viewer.slice(start, end)}; renderMemoryResources(memory)`, { + memory, + esc: (s: unknown) => String(s).replace(/[&<>"']/g, c => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'", + })[c]), }); +} +const mb = 1024 * 1024; +const base = { heapUsed: 45 * mb, heapTotal: 46 * mb, rss: 511.5 * mb }; +const evaluation = { + source: "heap", usedBytes: 45 * mb, limitBytes: 100 * mb, + percent: 45, severity: "healthy", available: true, +}; - it("computes the percentage from raw bytes, not the MB-rounded label values", () => { - expect(viewer).toMatch( - /heapPercent\s*=\s*ceilingBytes\s*>\s*0\s*\?\s*\(\(snap\.memory\.heapUsed\s*\|\|\s*0\)\s*\/\s*ceilingBytes\)\s*\*\s*100/, - ); - // The old form divided two already-rounded MB numbers. - expect(viewer).not.toMatch(/heapPct\s*=\s*heapTotal\s*>\s*0\s*\?\s*Math\.round\(\(heapUsed\s*\/\s*heapTotal\)/); +describe("viewer memory resources", () => { + it("uses server severity even when RSS rounds across the former floor", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, percent: 80.4, severity: "degraded" }] }); + expect(html).toContain("var(--yellow)"); + expect(html).toContain("80.4%"); + expect(html).not.toContain("var(--red)"); }); - - it("picks the gauge colour on the unrounded percentage", () => { - // Rounding first put the gauge a whole point out of step with - // evaluateHealth, which compares the raw value: at 80.4% health warns - // while a rounded 80 left the bar on the lower colour. - expect(viewer).toMatch(/heapPct\s*=\s*Math\.round\(heapPercent\)/); - expect(viewer).toMatch(/heapColor\s*=\s*\(heapPercent\s*>\s*80\s*&&\s*rssAboveFloor\)/); - expect(viewer).toMatch(/\(heapPercent\s*>\s*60\s*&&\s*rssAboveFloor\)/); + it("shows held recovery as critical rather than recoloring a low percentage", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, severity: "critical", transition: "recovering" }] }); + expect(html).toContain("var(--red)"); + expect(html).toContain("Recovery pending"); + }); + it("shows entering critical as pending and caps only the drawn width", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, percent: 102.2, severity: "degraded", transition: "entering" }] }); + expect(html).toContain("width:100%"); + expect(html).toContain("102.2%"); + expect(html).toContain("Critical pending"); + }); + it("renders old snapshots without an invented pressure limit", () => { + const html = render(base); + expect(html).toContain("Memory pressure unavailable"); + expect(html).toContain("Heap: 45 MiB"); + expect(html).not.toContain("var(--red)"); + }); + it("keeps unavailable latched signals visible and escapes their path", () => { + const html = render({ ...base, evaluations: [{ source: "cgroup-max", path: '', severity: "critical", available: false, transition: "unavailable" }] }); + expect(html).toContain("var(--red)"); + expect(html).toContain("Measurement unavailable"); + expect(html).toContain("<img"); + expect(html).not.toContain(" { + expect(render({ ...base, evaluations: [{ ...evaluation, source: "cgroup-high", severity: "degraded" }] })).toContain("Container throttle"); + }); +}); - it("rounds only for the displayed label", () => { - expect(viewer).toMatch(/heapCeiling\s*=\s*Math\.round\(ceilingBytes\s*\/\s*1024\s*\/\s*1024\)/); - expect(viewer).toMatch(/\+\s*heapUsed\s*\+\s*' \/ '\s*\+\s*heapCeiling\s*\+\s*' MB/); + +describe("viewer memory alerts", () => { + const start = viewer.indexOf(" function humanizeHealthFlag(f) {"); + const end = viewer.indexOf("\n }", start) + 6; + const humanize = (f: string) => runInNewContext(`${viewer.slice(start, end)}; humanizeHealthFlag(f)`, { f }); + it.each([ + ["memory_critical_heap_96%", "V8 heap", "96%"], + ["memory_warn_cgroup-high/_110%", "Container throttle", "110%"], + ["memory_entering_rss", "RSS budget", "Critical pending"], + ["memory_recovering_heap", "V8 heap", "Recovery pending"], + ["memory_unavailable_cgroup-max/parent", "Container limit", "Measurement unavailable"], + ])("describes %s", (slug, source, detail) => { + expect(humanize(slug)).toContain(source); + expect(humanize(slug)).toContain(detail); + expect(humanize(slug)).not.toBe(slug); + }); + it("keeps legacy snapshots readable", () => { + expect(humanize("memory_critical_97%_rss557mb")).toContain("process memory 557 MB"); }); }); From d16213721e2e0151a2904c08e8c2e9dde3ec3521 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 14:11:10 +0800 Subject: [PATCH 21/27] fix(cli): supervise the Docker engine container lifetime Wait for the exact iii-engine container after detached Compose startup. Preserve startup diagnostics, runtime exit policy, and observer cleanup. Signed-off-by: Omar Gerardo --- src/cli.ts | 33 ++++- src/cli/docker-engine.ts | 70 ++++++++++ test/docker-engine.test.ts | 264 +++++++++++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 src/cli/docker-engine.ts create mode 100644 test/docker-engine.test.ts diff --git a/src/cli.ts b/src/cli.ts index 6474a42949..05c8f6427a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -55,6 +55,7 @@ import { type RemoveOptions, } from "./cli/remove-plan.js"; import { createEngineLogForwarder } from "./cli/engine-log.js"; +import { watchDockerEngine } from "./cli/docker-engine.js"; import { renderSplash } from "./cli/splash.js"; import { isFirstRun, readPrefs, resetPrefs, writePrefs } from "./cli/preferences.js"; import { runOnboarding } from "./cli/onboarding.js"; @@ -917,6 +918,7 @@ type StartupFailure = { }; let startupFailure: StartupFailure | null = null; +let stopDockerEngineWatch: (() => void) | undefined; // Off by default. The engine's own stdout is the only place a frame's // `function_id` exists, but an unbounded log path here once wrote 137 GB @@ -948,6 +950,7 @@ function spawnEngineBackground( bin: string, spawnArgs: string[], label: string, + composeFile?: string, ): ChildProcess { vlog(`spawn: ${bin} ${spawnArgs.join(" ")}`); const child = spawn(bin, spawnArgs, { @@ -973,8 +976,13 @@ function spawnEngineBackground( stderrChunks.push(slice); stderrBytes += slice.length; }); - child.on("exit", (code, signal) => { + const onEngineExit = ( + code: number | null, + signal: NodeJS.Signals | null, + containerStopped = false, + ) => { const abnormal = + containerStopped || (code !== null && code !== 0) || (code === null && signal !== null); if (abnormal) { const stderr = Buffer.concat(stderrChunks).toString("utf-8"); @@ -1015,6 +1023,27 @@ function spawnEngineBackground( } } } + }; + child.on("exit", (code, signal) => { + if (isDocker && composeFile && code === 0) { + stopDockerEngineWatch?.(); + stderrChunks.length = 0; + stderrBytes = 0; + stopDockerEngineWatch = watchDockerEngine( + bin, + composeFile, + (containerCode) => onEngineExit(containerCode, null, true), + (error) => { + console.error(`[agentmemory] Docker engine supervision unavailable: ${error.message}`); + }, + ); + return; + } + onEngineExit(code, signal); + }); + child.on("error", (error) => { + stderrChunks.push(Buffer.from(error.message).subarray(0, MAX_STDERR_CAPTURE - stderrBytes)); + onEngineExit(1, null); }); child.unref(); return child; @@ -1190,6 +1219,7 @@ async function startEngine(): Promise { dockerBin, ["compose", "-f", composeFile, "up", "-d"], "iii-engine via Docker", + composeFile, ); s.stop("Docker compose started"); return true; @@ -2731,6 +2761,7 @@ async function stopDockerEngine(composeFile: string, port: number): Promise void, + onError: (error: Error) => void, +): () => void { + let child: ChildProcess | undefined; + let stopped = false; + const cancel = () => { + if (stopped) return; + stopped = true; + process.removeListener("exit", cancel); + child?.kill(); + }; + const fail = (error: Error) => { + if (stopped) return; + stopped = true; + process.removeListener("exit", cancel); + onError(error); + }; + process.once("exit", cancel); + try { + child = execFile( + dockerBin, + ["compose", "-f", composeFile, "ps", "--all", "--quiet", "iii-engine"], + { encoding: "utf8", timeout: 5_000, maxBuffer: 1024, windowsHide: true }, + (error, stdout) => { + if (stopped) return; + if (error) { fail(error); return; } + const containerId = stdout.trim(); + if (!/^[a-f0-9]{12,64}$/.test(containerId)) { + fail(new Error("Expected exactly one iii-engine container ID")); + return; + } + try { + const observer = spawn(dockerBin, ["wait", containerId], { + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + child = observer; + let output = ""; + observer.stdout.on("data", (chunk: Buffer) => { + output = (output + chunk.toString("utf8")).slice(0, 64); + }); + observer.on("error", fail); + observer.on("close", (code, signal) => { + if (stopped) return; + const containerCode = output.trim(); + if (code !== 0 || !/^\d{1,3}$/.test(containerCode) || Number(containerCode) > 255) { + fail(new Error(`docker wait failed (code=${code} signal=${signal})`)); + return; + } + stopped = true; + process.removeListener("exit", cancel); + onExit(Number(containerCode)); + }); + observer.unref(); + (observer.stdout as typeof observer.stdout & { unref?: () => void }).unref?.(); + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + }, + ); + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + return cancel; +} diff --git a/test/docker-engine.test.ts b/test/docker-engine.test.ts new file mode 100644 index 0000000000..fc8e9fc71f --- /dev/null +++ b/test/docker-engine.test.ts @@ -0,0 +1,264 @@ +import { EventEmitter } from "node:events"; +import { readFileSync } from "node:fs"; +import { PassThrough } from "node:stream"; +import { execFile, spawn } from "node:child_process"; +import { transpileModule } from "typescript"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { watchDockerEngine } from "../src/cli/docker-engine.js"; + +vi.mock("node:child_process", () => ({ spawn: vi.fn(), execFile: vi.fn() })); + +function child() { + return Object.assign(new EventEmitter(), { + pid: 123, + stdout: Object.assign(new PassThrough(), { unref: vi.fn() }), + stderr: new PassThrough(), + unref: vi.fn(), + kill: vi.fn(), + }); +} + +const source = readFileSync("src/cli.ts", "utf8"); +const functionStart = source.indexOf("function spawnEngineBackground("); +const functionEnd = source.indexOf("const ENGINE_STARTUP_GRACE_MS", functionStart); +const graceEnd = source.indexOf(";", functionEnd) + 1; +const compiledSpawn = transpileModule(source.slice(functionStart, graceEnd), {}).outputText; + +// Run the CLI's actual spawn function without executing its command dispatcher. +function cliSpawn() { + const exit = vi.fn(); + const clearState = vi.fn(); + const env: Record = {}; + const factory = new Function("spawn", "watchDockerEngine", "process", "clearEngineState", ` + const vlog = () => {}; + const writeEnginePidfile = () => {}; + const clearEnginePidfile = () => {}; + const ENGINE_LOG_ENABLED = false; + const IS_VERBOSE = false; + let startupFailure = null; + let stopDockerEngineWatch; + ${compiledSpawn} + return { + start: spawnEngineBackground, + failure: () => startupFailure, + stop: () => stopDockerEngineWatch?.(), + }; + `); + return { ...factory(spawn, watchDockerEngine, { env, exit }, clearState), exit, env, clearState }; +} + +type LookupCallback = (error: Error | null, stdout: string, stderr: string) => void; +let lookupCallback: LookupCallback; +let lookupChild: ReturnType; +function completeLookup(error: Error | null = null, ids = "a".repeat(64) + "\n") { + lookupCallback(error, ids, ""); +} +function finishStartup(compose: ReturnType) { + compose.emit("exit", 0, null); + completeLookup(); +} + +const cleanups: Array<() => void> = []; +beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(0); + lookupChild = child(); + vi.mocked(execFile).mockImplementation((...args: unknown[]) => { + lookupCallback = args[3] as LookupCallback; + return lookupChild as never; + }); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); +afterEach(() => { + for (const stop of cleanups.splice(0)) stop(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +function startDocker() { + const compose = child(); + const observer = child(); + vi.mocked(spawn).mockReturnValueOnce(compose as never).mockReturnValueOnce(observer as never); + const cli = cliSpawn(); + cleanups.push(cli.stop); + cli.start("docker", ["compose", "-f", "/own/compose.yml", "up", "-d"], "iii-engine via Docker", "/own/compose.yml"); + return { compose, observer, cli }; +} + +function containerExit(observer: ReturnType, code: number) { + observer.emit("exit", 0, null); + observer.stdout.emit("data", Buffer.from(`${code}\n`)); + observer.emit("close", 0, null); +} + +describe("Docker engine lifetime supervision", () => { + it("keeps timers responsive while identity lookup is pending", () => { + const { compose, cli } = startDocker(); + const tick = vi.fn(); + setTimeout(tick, 100); + compose.emit("exit", 0, null); + expect(execFile).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(100); + expect(tick).toHaveBeenCalledOnce(); + expect(cli.exit).not.toHaveBeenCalled(); + completeLookup(); + expect(spawn).toHaveBeenCalledTimes(2); + }); + + it.each([null, new Error("lookup cancelled")])("cancels pending lookup and ignores its late callback (%s)", (error) => { + const { compose, observer, cli } = startDocker(); + const before = process.listenerCount("exit"); + compose.emit("exit", 0, null); + expect(process.listenerCount("exit")).toBe(before + 1); + cli.stop(); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + expect(process.listenerCount("exit")).toBe(before); + completeLookup(error); + expect(spawn).toHaveBeenCalledTimes(1); + expect(compose.kill).not.toHaveBeenCalled(); + expect(observer.kill).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + expect(cli.exit).not.toHaveBeenCalled(); + cli.stop(); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + }); + + it("cancels pending lookup on parent exit", () => { + const { compose } = startDocker(); + const before = process.listeners("exit"); + compose.emit("exit", 0, null); + const onExit = process.listeners("exit").find((listener) => !before.includes(listener)); + expect(onExit).toBeDefined(); + onExit!(0); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + completeLookup(); + expect(spawn).toHaveBeenCalledTimes(1); + expect(process.listeners("exit")).toEqual(before); + }); + + it("cleans up when identity lookup throws synchronously", () => { + const { compose, cli } = startDocker(); + const before = process.listenerCount("exit"); + vi.mocked(execFile).mockImplementationOnce(() => { throw new Error("invalid executable"); }); + compose.emit("exit", 0, null); + expect(process.listenerCount("exit")).toBe(before); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("invalid executable")); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it.each([0, 137])("ignores detached client success, then reports actual container exit %i", (code) => { + const { compose, observer, cli } = startDocker(); + finishStartup(compose); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(execFile).toHaveBeenCalledWith("docker", [ + "compose", "-f", "/own/compose.yml", "ps", "--all", "--quiet", "iii-engine", + ], expect.objectContaining({ timeout: 5000 }), expect.any(Function)); + expect(spawn).toHaveBeenLastCalledWith("docker", ["wait", "a".repeat(64)], expect.anything()); + vi.setSystemTime(6000); + containerExit(observer, code); + expect(cli.exit).toHaveBeenCalledWith(1); + expect(cli.clearState).toHaveBeenCalledOnce(); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed" }); + }); + + it("keeps early container failure in startup diagnostics", () => { + const { compose, observer, cli } = startDocker(); + compose.stderr.emit("data", Buffer.from("Container iii-engine started")); + finishStartup(compose); + containerExit(observer, 137); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "process exited with code 137" }); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it("preserves compose startup failures without starting an observer", () => { + const { compose, cli } = startDocker(); + compose.stderr.emit("data", Buffer.from("image pull failed")); + compose.emit("exit", 1, null); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "image pull failed" }); + expect(spawn).toHaveBeenCalledTimes(1); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it("captures a Docker spawn error instead of leaving an unhandled error event", () => { + const { compose, cli } = startDocker(); + compose.emit("error", new Error("spawn ENOENT")); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "spawn ENOENT" }); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it("honors the engine-death opt-out", () => { + const { compose, observer, cli } = startDocker(); + cli.env.AGENTMEMORY_EXIT_ON_ENGINE_DEATH = "0"; + finishStartup(compose); + vi.setSystemTime(6000); + containerExit(observer, 137); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).toHaveBeenCalledOnce(); + }); + + it.each(["spawn", "exit", "output"])("reports observer %s errors without declaring the engine dead", (failure) => { + const { compose, observer, cli } = startDocker(); + finishStartup(compose); + vi.setSystemTime(6000); + if (failure === "spawn") observer.emit("error", new Error("spawn EACCES")); + observer.emit("close", failure === "output" ? 0 : 1, null); + expect(console.error).toHaveBeenCalledOnce(); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("supervision unavailable")); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.failure()).toBeNull(); + }); + + it.each(["", "a".repeat(64) + "\n" + "b".repeat(64)])("rejects missing or ambiguous engine identity", (ids) => { + const { compose, cli } = startDocker(); + compose.emit("exit", 0, null); + completeLookup(null, ids); + expect(spawn).toHaveBeenCalledTimes(1); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("exactly one iii-engine")); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it("reports identity lookup errors without stopping any container", () => { + const { compose, cli } = startDocker(); + const before = process.listenerCount("exit"); + compose.emit("exit", 0, null); + completeLookup(new Error("Docker unavailable")); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("Docker unavailable")); + expect(spawn).toHaveBeenCalledTimes(1); + expect(cli.exit).not.toHaveBeenCalled(); + expect(process.listenerCount("exit")).toBe(before); + }); + + it("reaps only the observer on intentional cleanup and allows the CLI to exit", () => { + const { compose, observer, cli } = startDocker(); + const before = process.listenerCount("exit"); + finishStartup(compose); + expect(process.listenerCount("exit")).toBe(before + 1); + expect(observer.unref).toHaveBeenCalledOnce(); + expect(observer.stdout.unref).toHaveBeenCalledOnce(); + cli.stop(); + expect(process.listenerCount("exit")).toBe(before); + expect(observer.kill).toHaveBeenCalledOnce(); + expect(compose.kill).not.toHaveBeenCalled(); + observer.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }); + + it("reaps the observer when its parent exits", () => { + const { compose, observer, cli } = startDocker(); + const before = process.listeners("exit"); + finishStartup(compose); + const onExit = process.listeners("exit").find((listener) => !before.includes(listener)); + expect(onExit).toBeDefined(); + onExit!(0); + expect(observer.kill).toHaveBeenCalledOnce(); + observer.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }); +}); From bfef6b5a41ebd4ae6b7f765e6a4e271d46c6a9e9 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 14:12:34 +0800 Subject: [PATCH 22/27] docs(health): align recovery policy and deployment guards Keep the implementation record consistent with opt-in KV escalation, five-second startup grace, and the pinned source-build flow. Signed-off-by: Omar Gerardo --- ...-infra-agentmemory-hang-resilience-plan.md | 49 +++++++++---------- test/deploy-entrypoint-drift.test.ts | 6 +-- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md index 7e67c83735..dcae6c7564 100644 --- a/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md +++ b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md @@ -422,12 +422,13 @@ setting to the repo root and repoint `dockerfilePath` accordingly. **Approach.** The current Dockerfile has two `COPY` lines, neither touching `src/`, and no build step. It installs the published package from npm, which is why `/opt/agentmemory/src` does not exist in the image and the deployed version -is 0.9.28. Change the build context to the repo root, copy the manifest and -lockfile, run `npm ci`, copy the source, run `npm run build` (`tsdown`), and point -the `agentmemory` bin at the built `dist/cli.mjs`. Prefer a multi-stage build so -dev dependencies do not ship. Add `.dockerignore` to keep `node_modules`, -`.git`, `dist`, and `eval/` out of the upload. Drop the vestigial `overrides` -block per KTD8, then verify the resolved `iii-sdk` rather than assuming. +is 0.9.28. Use the KTD6 source-build procedure: repository-root context, copy +`package.json`, install with npm 11.19.0 and `npm install` (the repository does +not commit a lockfile), copy source, build, then `npm pack`. Install that +tarball into `/opt/agentmemory` in the runtime stage, carrying the package +security overrides into the runtime manifest. Preserve the entrypoint +install layout and verify the resolved `iii-sdk` version. `.dockerignore` +excludes local dependencies, git metadata, build output, and evaluation data. Note that `dist/` is gitignored, so the image must build it and must never expect a prebuilt copy. @@ -493,7 +494,7 @@ detects it. **Approach.** The engine is spawned detached and already has a `child.on("exit")` handler that captures the exit code, the signal, and up to 16KB of stderr, then -logs under `vlog` and returns. Past a 60-second startup grace, report the death on +logs under `vlog` and returns. Past a 5-second startup grace, report the death on `console.error` with that stderr and exit non-zero. Deaths inside the grace keep the existing path, which renders a better startup message. @@ -604,28 +605,24 @@ the event loop still runs. claimed no health test file existed; that claim was wrong and acting on it destroyed the file once already. -**Approach.** `registerHealthMonitor` currently ends `collectHealth` with a -persist and a return. Add an in-memory consecutive-critical counter. **Increment -and evaluate it before the `await kv.set(KV.health, "latest", snapshot)` call at -`monitor.ts:94`, never after** — that persist is not raced against any timeout, -and during the real outage `state::set` hung for 180 seconds, so a counter behind -it would never advance during the failure it exists to catch (KTD7). On reaching -the threshold, log a clear reason and exit so `ON_FAILURE` restarts. +**Approach.** Keep escalation gated on consecutive KV probe failures and arm +it only after a healthy KV probe has been observed. Decide before the snapshot +persist, so a stalled write cannot postpone the decision. CPU, event-loop, +connection, and memory severity must not increment this counter. -Reuse the existing SIGTERM `shutdown` path at `src/index.ts:629-630` rather than -calling `process.exit` directly, so the KV store flushes. Fall back to a hard -exit only if shutdown does not complete within a grace window. - -Gate the whole behaviour behind an environment flag, default **off**, matching -U2's disabled-by-default posture and KTD5. +With `AGENTMEMORY_HEALTH_ESCALATE` enabled, ten consecutive failures after +arming request SIGTERM; force a nonzero exit after 15 seconds if shutdown +stalls. Railway uses `ALWAYS`, because successful graceful shutdown exits zero. +The feature remains off by default. **Test Scenarios.** -- N consecutive critical snapshots trigger exactly one escalation. -- A single healthy snapshot between criticals resets the counter. -- With the flag off, no escalation occurs regardless of snapshot status. -- The counter advances when the persist is slow or rejects. This is the - regression test for KTD7 and the one most likely to be got wrong. -- Escalation runs the graceful shutdown path, not a bare `process.exit`. +- Failures before any healthy KV probe never arm escalation. +- Ten consecutive KV failures after arming trigger exactly one escalation. +- A healthy KV probe resets the counter. +- Critical memory/CPU/lag with healthy KV never escalates. +- With the flag off, no escalation occurs. +- The decision precedes snapshot persistence, including stalled writes. +- Escalation requests graceful shutdown before the forced-exit fallback. **Verification.** `npm test`. Assert on the escalation decision, not on a real process exit. diff --git a/test/deploy-entrypoint-drift.test.ts b/test/deploy-entrypoint-drift.test.ts index 0fb2215032..21c3a9710f 100644 --- a/test/deploy-entrypoint-drift.test.ts +++ b/test/deploy-entrypoint-drift.test.ts @@ -39,10 +39,10 @@ describe("deploy entrypoint drift", () => { // reason lives in deploy/railway/entrypoint.sh next to the value, so deleting // this test cannot lose it. it("only railway disables iii-observability", () => { - expect(files.railway).toMatch(/enabled: false/); + expect(code(files.railway)).toMatch(/enabled: false/); for (const t of ["fly", "render", "coolify"] as const) { - expect(files[t]).toMatch(/enabled: true/); - expect(files[t]).not.toMatch(/enabled: false/); + expect(code(files[t])).toMatch(/enabled: true/); + expect(code(files[t])).not.toMatch(/enabled: false/); } }); From c8c4addc549d2c6d9f84e898703557f57065ce89 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 14:38:29 +0800 Subject: [PATCH 23/27] fix: enforce provider, logging, and cleanup bounds Recheck queued calls after semaphore acquisition without counting breaker rejections as provider failures. Measure partial logs in UTF-8 bytes and reclaim retired shards in batches of eight while preserving retry records. Signed-off-by: Omar Gerardo --- src/cli/engine-log.ts | 2 +- src/providers/resilient.ts | 17 +++++--- src/state/index-persistence.ts | 21 ++++++---- test/engine-log.test.ts | 8 ++++ test/index-persistence.test.ts | 73 ++++++++++++++++++++++++++++++++- test/resilient-provider.test.ts | 34 ++++++++++++++- 6 files changed, 139 insertions(+), 16 deletions(-) diff --git a/src/cli/engine-log.ts b/src/cli/engine-log.ts index 2dab376e05..773d3726df 100644 --- a/src/cli/engine-log.ts +++ b/src/cli/engine-log.ts @@ -93,7 +93,7 @@ export function createEngineLogForwarder( pending = pending.slice(newline + 1); newline = pending.indexOf("\n"); } - if (pending.length >= maxLineBytes) { + if (Buffer.byteLength(pending, "utf8") >= maxLineBytes) { emitRecord(pending); pending = ""; } diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index dd6c4dc223..bad018a464 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -73,12 +73,17 @@ export class ResilientProvider implements MemoryProvider { } await this.gate.acquire(); try { - const result = await fn(); - this.breaker.recordSuccess(); - return result; - } catch (err) { - if (!isRateLimited(err)) this.breaker.recordFailure(); - throw err; + if (!this.breaker.isAllowed) { + throw new Error("circuit_breaker_open"); + } + try { + const result = await fn(); + this.breaker.recordSuccess(); + return result; + } catch (err) { + if (!isRateLimited(err)) this.breaker.recordFailure(); + throw err; + } } finally { // In `finally` so a throw cannot leak the slot and deadlock every call // queued behind it. diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index bfe97a72cb..3fedd1a91e 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -551,13 +551,20 @@ export class IndexPersistence { continue; } const stranded: IndexGcLedger["generations"][number]["shards"] = []; - for (const shard of entry.shards) { - try { - await this.kv.delete(shard.scope, shard.key); - reclaimedPaths.push(statePath(shard.scope, shard.key)); - } catch { - failed += 1; - stranded.push(shard); + for (let offset = 0; offset < entry.shards.length; offset += 8) { + const batch = entry.shards.slice(offset, offset + 8); + const deleted = await Promise.all(batch.map(async shard => { + try { + await this.kv.delete(shard.scope, shard.key); + return true; + } catch { return false; } + })); + for (const [index, shard] of batch.entries()) { + if (deleted[index]) reclaimedPaths.push(statePath(shard.scope, shard.key)); + else { + failed += 1; + stranded.push(shard); + } } } if (stranded.length > 0) { diff --git a/test/engine-log.test.ts b/test/engine-log.test.ts index 6b839e1482..313b32d609 100644 --- a/test/engine-log.test.ts +++ b/test/engine-log.test.ts @@ -152,6 +152,14 @@ describe("engine log forwarder", () => { expect(h.lines).toEqual(["[engine] 0123456789abcdefghij"]); }); + it("flushes a partial UTF-8 record at its byte limit", () => { + const h = harness({ maxLineBytes: 6 }); + h.push("☃"); + expect(h.lines).toEqual([]); + h.push("☃"); + expect(h.lines).toEqual(["[engine] ☃☃"]); + }); + it("ships ceilings that bound a firehose without operator action", () => { expect(ENGINE_LOG_MAX_TOTAL_BYTES).toBe(32 * 1024 * 1024); expect(ENGINE_LOG_MAX_BYTES_PER_SECOND).toBe(64 * 1024); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 3dff054091..4da40fd6f4 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; -import type { CompressedObservation } from "../src/types.js"; +import type { AuditEntry, CompressedObservation } from "../src/types.js"; const BM25_SCOPE = "mem:index:bm25"; const BM25_LEGACY_KEY = "data"; @@ -629,6 +629,77 @@ describe("IndexPersistence", () => { await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); }); + it("bounds concurrent reclaim deletes and retries failures without touching live or future shards", async () => { + await new IndexPersistence(kv as never, makeBm25("obs_live", "live snapshot"), null, { + createGeneration: () => "gen_live", + }).save(); + const live = await getBm25Manifest(kv); + const retired = Array.from({ length: 19 }, (_, i) => ({ scope: `retired:${i}`, key: "data" })); + const future = { scope: "future:0", key: "data" }; + for (const shard of [...retired, future]) await kv.set(shard.scope, shard.key, "shard"); + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = { + v: 1, + generations: [ + { generation: "gen_old", shards: retired }, + { generation: "gen_live", shards: live.shards.map(({ scope, key }) => ({ scope, key })) }, + { generation: "gen_future", shards: [future] }, + ], + }; + await kv.set(BM25_SCOPE, gcKey, ledger); + const failedScopes = new Set([retired[1].scope, retired[10].scope]); + const pending: Array<() => void> = []; + let active = 0; + let peak = 0; + const boundedKv = { + ...kv, + delete: vi.fn(async (scope: string, key: string) => { + active++; + peak = Math.max(peak, active); + try { + await new Promise(resolve => pending.push(resolve)); + if (failedScopes.has(scope)) throw new Error("retry delete"); + await kv.delete(scope, key); + } finally { active--; } + }), + }; + const loading = new IndexPersistence(boundedKv as never, new SearchIndex(), null).load(); + for (const size of [8, 8, 3]) { + await vi.waitFor(() => expect(pending).toHaveLength(size)); + expect(active).toBe(size); + for (const resolve of pending.splice(0).reverse()) resolve(); + } + const loaded = await loading; + expect(peak).toBe(8); + expect(loaded.bm25!.search("live")).toHaveLength(1); + expect(boundedKv.delete.mock.calls.map(([scope, key]) => ({ scope, key }))).toEqual(retired); + expect(await kv.get(BM25_SCOPE, gcKey)).toEqual({ + v: 1, + generations: [ + { generation: "gen_old", shards: [retired[1], retired[10]] }, + ...ledger.generations.slice(1), + ], + }); + const sweeps = (await kv.list("mem:audit")) + .filter(entry => entry.details.reason === "generation_reclaim"); + expect(sweeps).toHaveLength(1); + expect(sweeps[0].details).toMatchObject({ evicted: 17, failed: 2 }); + expect(sweeps[0].targetIds).toEqual(retired + .filter(shard => !failedScopes.has(shard.scope)) + .map(shard => `${shard.scope}/${shard.key}`)); + + const retryKv = { ...kv, delete: vi.fn(kv.delete) }; + await new IndexPersistence(retryKv as never, new SearchIndex(), null).load(); + expect(retryKv.delete.mock.calls).toEqual([ + [retired[1].scope, "data"], [retired[10].scope, "data"], + ]); + expect(await kv.get(BM25_SCOPE, gcKey)).toEqual({ v: 1, generations: ledger.generations.slice(1) }); + for (const shard of retired) expect(await kv.get(shard.scope, shard.key)).toBeNull(); + for (const shard of [...live.shards, future]) { + expect(await kv.get(shard.scope, shard.key)).not.toBeNull(); + } + }); + it("reclaims the previous vector generation when the vector manifest read fails (#1115)", async () => { await new IndexPersistence( kv as never, diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts index 4f7b028a13..1b09e21c0c 100644 --- a/test/resilient-provider.test.ts +++ b/test/resilient-provider.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { ResilientProvider } from "../src/providers/resilient.js"; import type { MemoryProvider } from "../src/types.js"; @@ -66,6 +66,38 @@ describe("ResilientProvider concurrency", () => { expect(settled).toHaveLength(5); expect(settled.filter((s) => s.status === "fulfilled")).toHaveLength(3); }); + + it("rejects queued calls when earlier failures open the breaker without recording new failures", async () => { + const inner = countingProvider(async n => { + if (n <= 3) throw new Error("upstream exploded"); + return "recovered"; + }); + const provider = new ResilientProvider(inner, { maxConcurrent: 1 }); + let now = 1000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now++); + try { + const settled = await Promise.allSettled( + Array.from({ length: 6 }, (_, i) => i % 2 + ? provider.summarize("sys", "user") : provider.compress("sys", "user")), + ); + + expect(inner.calls).toBe(3); + expect(settled.map(result => result.status === "rejected" ? result.reason.message : result.value)).toEqual([ + "upstream exploded", "upstream exploded", "upstream exploded", + "circuit_breaker_open", "circuit_breaker_open", "circuit_breaker_open", + ]); + expect(provider.circuitState).toEqual({ + state: "open", failures: 3, lastFailureAt: 1002, openedAt: 1002, + }); + + now = 31_002; + await expect(provider.compress("sys", "user")).resolves.toBe("recovered"); + expect(inner.calls).toBe(4); + expect(provider.circuitState.state).toBe("closed"); + } finally { + clock.mockRestore(); + } + }); }); describe("ResilientProvider rate limiting", () => { From fee13e0e25e29945c5a89ddcbe9e640ccbb75c9c Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 14:50:13 +0800 Subject: [PATCH 24/27] docs(railway): record reproducible memory health validation (U4) Record matched baseline/candidate runs, held entry and recovery, file-cache capacity behavior, immutable artifacts, operational correction, and cleanup. Signed-off-by: Omar Gerardo --- deploy/railway/health-validation.md | 196 ++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 deploy/railway/health-validation.md diff --git a/deploy/railway/health-validation.md b/deploy/railway/health-validation.md new file mode 100644 index 0000000000..f6100d0ea1 --- /dev/null +++ b/deploy/railway/health-validation.md @@ -0,0 +1,196 @@ +# Health memory validation on Railway + +This experiment uses the existing `agentmemory` project and its isolated `sandbox` environment. It does not use production data or Railway's ephemeral sandbox product. + +## Environment and artifacts + +- Isolated `sandbox` deployment environment; no production data used. +- Linux cgroup v2, Node `v22.23.2`, iii engine and iii-sdk `0.11.2`. +- One Singapore replica, two vCPUs, `memory.max=8589934592` (8 GiB), `memory.high=max`, swap disabled. +- `NODE_OPTIONS=--max-old-space-size=6144`; observed V8 `heap_size_limit=6492782592`. +- Node, iii and the synthetic sibling allocator all report `0::/`. The visible cgroup mount is the namespace root. Hidden ancestor controls cannot be inspected from this container. +- Fresh ephemeral `/data`; the retained volume remained unattached and was never mounted by experimental code. +- No LLM/embedding API credentials, embedding provider or embedding shim: successful observation writes validate the provider-free path. +- Platform healthcheck remains `/agentmemory/livez`, with a 300-second startup timeout. + +The tested candidate runtime is commit `d16213721e2e0151a2904c08e8c2e9dde3ec3521`. Later commits include documentation and unrelated log/provider/GC changes validated separately by local regression tests; this live experiment does not claim their complete bundle matches the measured image. The baseline and current variants use source `42bcdce85eacb770ecf45c7315168f8be312d0b4`. The baseline changes only `heapCeiling` back to `snapshot.memory.heapTotal` in `evaluateHealth`, recreating the defect without comparing unrelated releases. Test-only code and builds stay outside the PR. + +Runtime dependency lock SHA-256: `9030232b4f6092ec74f351b61859b99f1cbf9e22a7bfe09ecce11152d6668ed2`. Subsequent builds use this lock; compare the resolved dependency version maps, because package tarball integrity changes with source. Record each image digest and installed `dist/index.mjs` hash separately. + +## Workload + +A test-only module imported by the application worker retains 640 MiB of JavaScript numbers in 8 MiB arrays, retains a touched 64 MiB Buffer, and starts a sibling Node process with another touched 64 MiB Buffer. Each second it replaces an 8 MiB array to create ordinary allocation/GC churn. The sibling's cgroup membership is verified rather than assumed. + +After at least 65 seconds of warm-up, take 31 samples at 30-second intervals: a full 15-minute first-to-last window. For every sample, capture `/health` HTTP status and JSON from the same response; capture `/livez`, a successful synthetic `/observe` write, worker memory, V8 limit, cgroup current/max/high/swap, memory.events and PSI. Verify health `uptimeSeconds` advances so repeated polls cannot masquerade as fresh collection. + +The initial single-array experiment is excluded from the comparison. One very large JS array caused transient V8 allocation overhead; every accepted run uses the same bounded 8 MiB chunks. + +## Results + +| Variant | Fresh samples / duration | Health | Liveness / observations | Heap used / committed | Heap used / V8 limit | +|---|---|---|---|---|---| +| Reintroduced baseline | 31 / 15m0.406s | 31 × 503, memory-critical | 31 × 200 / 31 × 201 | 95.058–96.972% | 10.807–18.682% | +| Existing PR #1224 | 31 / 15m10.920s | 31 × 200, healthy | 31 × 200 / 31 × 201 | 95.031–97.670% | 10.808–23.731% | +| Candidate `d16213721e2e0151a2904c08e8c2e9dde3ec3521` | 31 / 15m0.409s | 31 × 200, healthy | 31 × 200 / 31 × 201 | 95.203–97.482% | 10.939–19.725% | + +All three variants used identical Node/iii versions and the same 232 resolved package versions. Every paired response had matching body status and HTTP code, every health uptime was unique, and no cgroup memory event or PSI total increased. + +| Variant | Image digest | Installed `dist/index.mjs` SHA-256 | +|---|---|---| +| Baseline | `sha256:f48371808108061acb706f71760f082868919865871d1a65f91b0ad88318ef43` | `f555f9f2558e4a6f1d7795fe95ea904559d1954f719a68a7becdb3146421a048` | +| Current PR | `sha256:13b14bbb5d24ad737cee9b16ad134475a1c64b0509585873a826209ff8f63abb` | `e824533a643bf28b96eb110d8b51822c5f1fabbc33f85c7f349f57a44bccac46` | +| Candidate | `sha256:2fadeee1e7995a6ca4d90654022c15b7999ef2f63a196c814276fe89eb668d14` | `ec8af71563960e3ca0f4bd4c34d3a0c15d3e915b026a2e0e05e5d6c999610459` | + +Workload module SHA-256: `f8ace48d97343665518729861de6506deb80cdff7142489266382cce27676f9e`. + +## Controlled transition and file-cache checks + +The hold test produced 97 paired polls over eight fresh collector snapshots. Both heap and cgroup-max entered pending pressure at 06:38:34 UTC (degraded, HTTP 200, sample 1/2), became critical at 06:39:03 (HTTP 503), remained critical on the first clear snapshot at 06:40:04 (recovering, sample 1/2), and recovered at 06:40:33 (healthy, HTTP 200). Repeated polls within a snapshot did not advance either transition counter. Both runs use the same candidate image with sandbox-only warning/critical thresholds of 3%/5%, keeping actual use far below the 8 GiB cap. Preliminary transition recovery and cache output is excluded: the first test-only forced-GC helper used unsupported dynamic import in an inspector expression. The corrected helper calls `HeapProfiler.collectGarbage`; application code and the allocation module are unchanged. Clearing the control file drops bulk heap/native/sibling allocations but retains the last 8 MiB churn array until process stop, explaining the approximately 31 MiB recovery heap floor. + +The separate file-cache test produced 97 paired polls over seven fresh snapshots. Writing a 512 MiB disposable file moved cgroup occupancy to approximately 7.5% while heap use stayed approximately 0.5%. Cgroup-max entered pending pressure at 06:42:03 UTC, became critical at 06:42:33, entered recovery after unlinking the file at 06:43:34, and returned healthy at 06:44:04. Heap remained healthy throughout. Every controlled poll returned liveness 200 and observation 201; all body/HTTP mappings matched, and memory.events, PSI totals and swap remained zero. + +Controlled-phase helper SHA-256: + +- `force-gc.mjs`: `938c878c2158de38b2c55a1f36434ccd1444411747a35f9b49f1109dd26a44e9` +- `hold-samples.mjs`: `5453cdb068d5ed6fd0657ec23c57425ce658350c77931af221c6ad7921ffec52` +- `cache-samples.mjs`: `95f3948ddcd60f7303b0a57cdb7b99ce3b13f0abeb6180132145f95c0aa72321` + +### Operational correction + +Railway CLI 5.27 treats service source disconnect as service-wide even with an environment argument. Using it during sandbox setup removed a shared production source binding. It was restored with an environment-scoped patch and `skipDeploys:true`; configuration read-back and unchanged production deployment identities verified the correction without a production deployment. Private incident records retain the exact scope and evidence. + +## Limitations + +This is a synthetic workload on a fresh store. It does not establish retained-corpus capacity or production startup performance. The live environment exposes neither finite `memory.high` nor nested systemd ancestor controls; those cases are covered by filesystem fixtures. Cgroup v1 and hidden ancestors remain unsupported/unobservable as documented. Cgroup filesystem reads have no explicit timeout; the Docker engine observer itself has no retry mechanism. Lowered thresholds test transitions without deliberately approaching physical OOM. `memory.current` includes reclaimable file cache: current/max evaluates configured capacity occupancy, not proof that an allocation would fail or that OOM is imminent. + +## Cleanup + +Bulk synthetic allocations and the disposable cache file were removed, and test-only threshold overrides were deleted with `skipDeploys:true`. Railway normalized a last-region scale-to-zero request into its default region with one configured replica; the resulting sandbox deployment was canceled and the measured sandbox deployment stopped explicitly. Final read-back verifies no active sandbox deployment or running instance. The final 8 MiB churn array disappears with process stop. Service/configuration and artifact evidence remain; the retained volume stays READY and unattached. Production source/trigger configuration and both production deployment identities remain unchanged. Configured replica defaults remain present; this cleanup establishes an inactive sandbox rather than claiming a persisted zero-replica configuration. + +## Reproducing the deterministic checks + +```sh +npx vitest run test/health-memory.test.ts test/health-collection.test.ts test/health-monitor.test.ts test/health-thresholds.test.ts +npm test +npm run build +``` + +For another live comparison, export each recorded source to its own build context, reuse the dependency lock, add the same test-only worker allocation module, and deploy only to an isolated environment. Keep provider configuration, V8 limit, cgroup cap, replica count and warm-up unchanged. Test code must never be imported by a production build. API authentication must be read inside the container; do not print the generated HMAC secret. + +At the end of steady-state sampling, redeploy the exact candidate image with sandbox-only `AGENTMEMORY_HEALTH_MEM_WARN_PCT=3` and `AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=5`. Start before bulk test allocations, then apply the bounded workload. Polling faster than the collector must leave each snapshot's transition count unchanged. Remove bulk heap/native/sibling allocations and explicitly GC the worker only for this controlled test; verify recovery requires two fresh collector samples. Delete the test-only threshold overrides afterward. + +### Exact test-only allocation module + +In an isolated source export, save the following as `src/repro-runtime.ts` and add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module is inert until its control file requests allocations. + +```typescript +import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; +import { getHeapStatistics } from 'node:v8'; +import { spawn, type ChildProcess } from 'node:child_process'; +let heap: number[][] = []; +let native: Buffer | undefined; +let child: ChildProcess | undefined; +let previous = ''; +let churn: number[] = []; +const read = (p: string) => { try { return readFileSync(p, 'utf8').trim(); } catch { return null; } }; +setInterval(() => { + const control = read('/tmp/health-repro-control.json') || '{}'; + if (control !== previous) { + const config = JSON.parse(control); + heap = Array.from({length:Math.ceil((config.heapMiB || 0)/8)},()=>new Array(1024*1024).fill(0.25)); + native = config.nativeMiB ? Buffer.alloc(config.nativeMiB * 1024 * 1024, 0x5a) : undefined; + child?.kill(); child = undefined; + if (config.siblingMiB) child = spawn(process.execPath, ['-e', `const b=Buffer.alloc(${config.siblingMiB}*1024*1024,0x55);setInterval(()=>{if(!b[0])process.exit(1)},1000)`], { stdio: 'ignore', env: { ...process.env, NODE_OPTIONS: '' } }); + previous = control; + } + if (heap.length) churn = new Array(1024 * 1024).fill(Math.random()); + const processes = readdirSync('/proc').filter(p => /^\d+$/.test(p)).map(pid => ({pid, comm:read(`/proc/${pid}/comm`), cgroup:read(`/proc/${pid}/cgroup`), status:read(`/proc/${pid}/status`)?.split('\n').filter(l=>/^(VmRSS|VmSize|Name|PPid):/.test(l))})); + const cgroup = Object.fromEntries(['memory.current','memory.max','memory.high','memory.events','memory.pressure','memory.swap.current','memory.swap.max'].map(f => [f,read(`/sys/fs/cgroup/${f}`)])); + writeFileSync('/tmp/health-repro-diagnostics.json',JSON.stringify({timestamp:new Date().toISOString(),deploymentId:process.env.RAILWAY_DEPLOYMENT_ID,variant:process.env.HEALTH_REPRO_VARIANT,pid:process.pid,node:process.version,versions:process.versions,nodeOptions:process.env.NODE_OPTIONS,memory:process.memoryUsage(),heapSizeLimit:getHeapStatistics().heap_size_limit,allocations:{heapElements:heap.length * 1024 * 1024,nativeBytes:native?.length||0,churnElements:churn.length,siblingPid:child?.pid},cgroup,processes,mountinfo:read('/proc/self/mountinfo')?.split('\n').filter(l=>l.includes('cgroup'))})); +}, 1000).unref(); +``` + +Save the following as `/tmp/health-repro-sample.mjs` inside the sandbox container. It reads authentication locally and prints only response/diagnostic evidence. + +```javascript +import {readFileSync,writeFileSync} from 'node:fs'; +const secret=readFileSync('/data/.hmac','utf8').trim(); +const variant=process.argv[2]||'current'; +const count=Number(process.argv[3]||30); +const headers={authorization:`Bearer ${secret}`,'content-type':'application/json'}; +const request=async(path,body)=>{ + const at=new Date().toISOString();const response=await fetch('http://127.0.0.1:3111/agentmemory/'+path,{method:body?'POST':'GET',headers,body:body?JSON.stringify(body):undefined,signal:AbortSignal.timeout(15000)}); + return {at,status:response.status,body:await response.json()}; +}; +for(let n=0;nsetTimeout(r,Math.max(0,30000-(Date.now()-start)))); +} +``` + +After verifying the exact deployment is successful, run inside that container: + +```sh +node - <<'NODE' +const fs = require("node:fs"); +fs.writeFileSync("/tmp/health-repro-control.next", JSON.stringify({ heapMiB: 640, nativeMiB: 64, siblingMiB: 64 })); +fs.renameSync("/tmp/health-repro-control.next", "/tmp/health-repro-control.json"); +NODE +sleep 65 +node /tmp/health-repro-sample.mjs candidate 31 > /tmp/health-samples.jsonl +node - <<'NODE' +const fs = require("node:fs"); +fs.writeFileSync("/tmp/health-repro-control.next", "{}"); +fs.renameSync("/tmp/health-repro-control.next", "/tmp/health-repro-control.json"); +NODE +``` + +Changing the label only identifies the output; select baseline/current/candidate by deploying the corresponding source artifact. Archive the JSONL before stopping the disposable container. The baseline-only source change is `const heapCeiling = snapshot.memory.heapTotal;` in the recorded old `evaluateHealth` implementation; do not apply it to the candidate implementation. + +### Controlled checks (reproducible pseudocode) + +The following sequence describes the transition samplers. Use the same request helper and diagnostics as above; poll every two seconds while the application collector runs every 30 seconds. Every poll includes one health response, one liveness response and one synthetic observation write. + +```text +Redeploy identical candidate image with sandbox-only warning=3, critical=5. +Wait at least 70 seconds with control={} and verify healthy. +Record before sample. +Atomically set control={heapMiB:640,nativeMiB:64,siblingMiB:64}. +Record 48 entry polls, two seconds apart. +Atomically set control={heapMiB:0,nativeMiB:0,siblingMiB:0}; wait two seconds. +Force worker GC via inspector HeapProfiler.collectGarbage (helper below). +Record 48 recovery polls, two seconds apart. +Verify entry: entering(samples=1)/200, then critical/503 on next fresh snapshot. +Verify recovery: recovering(samples=1)/503, then healthy/200 on next fresh snapshot. +Verify repeated polls with identical uptimeSeconds never change transition state. + +Starting healthy after removing bulk heap/native/sibling load: +Record before sample. +Write /tmp/health-repro-cache.bin using 512 writes of one touched 1 MiB buffer. +Record 48 file-cache polls, two seconds apart; include memory.stat. +Unlink only that test file. +Record 48 cache-recovery polls, two seconds apart. +Verify cgroup occupancy can independently enter/recover while heap stays healthy. +``` + +The corrected GC helper runs only inside the disposable container. It opens the worker's loopback inspector, collects dead test allocations, records memory and closes the inspector. Save as `/tmp/force-gc.mjs`: + +```javascript +import {readFileSync} from 'node:fs'; +const pid=JSON.parse(readFileSync('/tmp/health-repro-diagnostics.json','utf8')).pid; +process.kill(pid,'SIGUSR1'); +await new Promise(r=>setTimeout(r,1000)); +const targets=await (await fetch('http://127.0.0.1:9229/json/list')).json(); +const ws=new WebSocket(targets[0].webSocketDebuggerUrl); +await new Promise((resolve,reject)=>{ws.addEventListener('open',resolve,{once:true});ws.addEventListener('error',reject,{once:true});}); +let id=0; +async function command(method,params={}){const requestId=++id;return new Promise((resolve,reject)=>{const timer=setTimeout(()=>reject(new Error('Inspector timeout')),10000);function handle(e){const r=JSON.parse(e.data);if(r.id!==requestId)return;clearTimeout(timer);ws.removeEventListener('message',handle);if(r.error||r.result?.exceptionDetails)reject(new Error(JSON.stringify(r)));else resolve(r.result);}ws.addEventListener('message',handle);ws.send(JSON.stringify({id:requestId,method,params}));});} +await command('HeapProfiler.collectGarbage'); +console.log(JSON.stringify({forcedGc:await command('Runtime.evaluate',{expression:'process.memoryUsage()',returnByValue:true})})); +await command('Runtime.evaluate',{expression:'setTimeout(() => process.getBuiltinModule("inspector").close(), 100)'}); +ws.close(); +``` From 8b2699bbd60f806132ca89163e518b2813fc7d75 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 15:02:20 +0800 Subject: [PATCH 25/27] docs(railway): include the exact file-cache sampler Capture memory.stat in the cache helper without changing the recorded allocation module or its artifact hash. Signed-off-by: Omar Gerardo --- deploy/railway/health-validation.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/deploy/railway/health-validation.md b/deploy/railway/health-validation.md index f6100d0ea1..9f8aa05292 100644 --- a/deploy/railway/health-validation.md +++ b/deploy/railway/health-validation.md @@ -81,7 +81,7 @@ At the end of steady-state sampling, redeploy the exact candidate image with san ### Exact test-only allocation module -In an isolated source export, save the following as `src/repro-runtime.ts` and add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module is inert until its control file requests allocations. +In an isolated source export, save the following as `src/repro-runtime.ts` and add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module collects diagnostics every second and adds bulk allocations only when its control file requests them. ```typescript import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; @@ -177,6 +177,22 @@ Record 48 cache-recovery polls, two seconds apart. Verify cgroup occupancy can independently enter/recover while heap stays healthy. ``` +The file-cache sampler reads `memory.stat` directly for each poll, separately from the unchanged allocation module. Save this exact helper as `/tmp/cache-samples.mjs` and run `node /tmp/cache-samples.mjs` inside the disposable container after recovery: + +```javascript +import {readFileSync,openSync,writeSync,closeSync,unlinkSync} from 'node:fs'; +const secret=readFileSync('/data/.hmac','utf8').trim(); +const headers={authorization:`Bearer ${secret}`,'content-type':'application/json'}; +const request=async(path,body)=>{const at=new Date().toISOString();const response=await fetch('http://127.0.0.1:3111/agentmemory/'+path,{method:body?'POST':'GET',headers,body:body?JSON.stringify(body):undefined,signal:AbortSignal.timeout(15000)});return {at,status:response.status,body:await response.json()};}; +const sample=async(phase,n)=>{const health=await request('health');const livez=await request('livez');const observe=await request('observe',{hookType:'PostToolUse',sessionId:'health-repro-cache',project:'health-repro',cwd:'/synthetic',timestamp:new Date().toISOString(),data:{tool_name:'Read',tool_input:{file_path:'/synthetic/cache-'+phase+'-'+n+'.txt'},tool_response:'Synthetic cache capacity sample'}});console.log(JSON.stringify({phase,n,deploymentId:process.env.RAILWAY_DEPLOYMENT_ID,health,livez,observe,diagnostics:JSON.parse(readFileSync('/tmp/health-repro-diagnostics.json','utf8')),memoryStat:readFileSync('/sys/fs/cgroup/memory.stat','utf8')}));}; +await sample('before',0); +const fd=openSync('/tmp/health-repro-cache.bin','w');const buffer=Buffer.alloc(1024*1024,0x5a); +for(let n=0;n<512;n++)writeSync(fd,buffer);closeSync(fd); +for(let n=0;n<48;n++){await sample('file-cache',n);await new Promise(r=>setTimeout(r,2000));} +unlinkSync('/tmp/health-repro-cache.bin'); +for(let n=0;n<48;n++){await sample('cache-recovery',n);await new Promise(r=>setTimeout(r,2000));} +``` + The corrected GC helper runs only inside the disposable container. It opens the worker's loopback inspector, collects dead test allocations, records memory and closes the inspector. Save as `/tmp/force-gc.mjs`: ```javascript From 8075d09f3784a6e2e67c4234c93c8172623bf2df Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 15:16:59 +0800 Subject: [PATCH 26/27] docs(railway): publish reproduction diagnostics atomically Signed-off-by: Omar Gerardo --- deploy/railway/health-validation.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/deploy/railway/health-validation.md b/deploy/railway/health-validation.md index 9f8aa05292..68a9a1c044 100644 --- a/deploy/railway/health-validation.md +++ b/deploy/railway/health-validation.md @@ -81,7 +81,7 @@ At the end of steady-state sampling, redeploy the exact candidate image with san ### Exact test-only allocation module -In an isolated source export, save the following as `src/repro-runtime.ts` and add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module collects diagnostics every second and adds bulk allocations only when its control file requests them. +The following is the frozen module used for the recorded measurements and workload hash above. In an isolated source export, save it as `src/repro-runtime.ts`, apply the required publication patch below, then add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module collects diagnostics every second and adds bulk allocations only when its control file requests them. ```typescript import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; @@ -110,6 +110,32 @@ setInterval(() => { }, 1000).unref(); ``` +### Required diagnostics publication patch for new runs + +The measured module truncates the diagnostics file while rewriting it, so another process can read partial JSON. Before building any new experiment, run this patch in its isolated source export. It writes diagnostics to a temporary file in the same directory, then atomically renames it over the reader-visible path. All diagnostics readers then receive a complete previous or current snapshot. + +This changes only diagnostics publication. The original module, hash and recorded measurements above remain historical evidence; the patched module has a different hash and has not been measured in those runs. Record the printed hash with each new run and use the patched module consistently across its variants. + +```sh +node --input-type=module <<'NODE' +import { readFileSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +const file = 'src/repro-runtime.ts'; +let source = readFileSync(file, 'utf8'); +if (createHash('sha256').update(source).digest('hex') !== 'f8ace48d97343665518729861de6506deb80cdff7142489266382cce27676f9e') { + throw new Error('Expected the exact measured allocation module before patching'); +} +source = source + .replace('writeFileSync, readdirSync', 'writeFileSync, renameSync, readdirSync') + .replace("writeFileSync('/tmp/health-repro-diagnostics.json',", "writeFileSync('/tmp/health-repro-diagnostics.next',") + .replace('\n}, 1000).unref();', "\n renameSync('/tmp/health-repro-diagnostics.next', '/tmp/health-repro-diagnostics.json');\n}, 1000).unref();"); +writeFileSync(file, source); +console.log('Atomic-publication workload SHA-256:', createHash('sha256').update(source).digest('hex')); +NODE +``` + +### Steady-state sampler + Save the following as `/tmp/health-repro-sample.mjs` inside the sandbox container. It reads authentication locally and prints only response/diagnostic evidence. ```javascript From 04f555c47ce142700f0fcc9dc1daa8c29cd912c4 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 16 Sep 2026 16:17:51 +0800 Subject: [PATCH 27/27] fix(health): require a successful KV probe before escalation Signed-off-by: Omar Gerardo --- src/health/monitor.ts | 5 +++-- test/docker-engine.test.ts | 3 +++ test/health-monitor.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/health/monitor.ts b/src/health/monitor.ts index f517a7202d..4a8d0cd420 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -38,12 +38,13 @@ export function bumpEscalation( state: EscalationState, threshold: number, ): boolean { - const stalled = snapshot.kvConnectivity?.status === "error"; - if (!stalled) { + const status = snapshot.kvConnectivity?.status; + if (status === "ok") { state.hasSeenHealthyKvProbe = true; state.consecutiveKvProbeFailures = 0; return false; } + if (status !== "error") return false; if (!state.hasSeenHealthyKvProbe) return false; state.consecutiveKvProbeFailures += 1; if (state.escalated || state.consecutiveKvProbeFailures < threshold) return false; diff --git a/test/docker-engine.test.ts b/test/docker-engine.test.ts index 42fc8fbaf2..b43a0e967c 100644 --- a/test/docker-engine.test.ts +++ b/test/docker-engine.test.ts @@ -22,8 +22,11 @@ function child() { const source = readFileSync("src/cli.ts", "utf8"); const functionStart = source.indexOf("function createEngineExitHandler("); +expect(functionStart, "CLI createEngineExitHandler source marker").toBeGreaterThanOrEqual(0); const functionEnd = source.indexOf("const ENGINE_STARTUP_GRACE_MS", functionStart); +expect(functionEnd, "CLI engine startup grace source marker").toBeGreaterThan(functionStart); const graceEnd = source.indexOf(";", functionEnd) + 1; +expect(graceEnd, "CLI engine startup grace declaration terminator").toBeGreaterThan(functionEnd); const compiledSpawn = transpileModule(source.slice(functionStart, graceEnd), {}).outputText; // Run the CLI's actual spawn function without executing its command dispatcher. diff --git a/test/health-monitor.test.ts b/test/health-monitor.test.ts index 54bbc8bbfb..f91d104982 100644 --- a/test/health-monitor.test.ts +++ b/test/health-monitor.test.ts @@ -18,6 +18,13 @@ function snap(over: Partial = {}): HealthSnapshot { } const stalled = () => snap({ kvConnectivity: { status: "error", error: "kv_probe_failed" } }); +const noSignalProbes: Array<[string, unknown]> = [ + ["absent", undefined], + ["null", null], + ["missing status", {}], + ["unknown status", { status: "pending" }], + ["malformed status", { status: 123 }], +]; function fresh(): EscalationState { return { consecutiveKvProbeFailures: 0, hasSeenHealthyKvProbe: false, escalated: false }; @@ -48,6 +55,15 @@ describe("bumpEscalation arming", () => { bumpEscalation(snap(), state, 3); expect(state.hasSeenHealthyKvProbe).toBe(true); }); + + it.each(noSignalProbes)("does not arm from %s probe data or subsequent errors", (_label, probe) => { + const state = fresh(); + expect(bumpEscalation(snap({ kvConnectivity: probe as HealthSnapshot["kvConnectivity"] }), state, 3)).toBe(false); + for (let i = 0; i < 10; i++) { + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + } + expect(state).toEqual(fresh()); + }); }); describe("bumpEscalation gating", () => { @@ -67,6 +83,16 @@ describe("bumpEscalation gating", () => { expect(bumpEscalation(stalled(), state, 3)).toBe(false); }); + it.each(noSignalProbes)("preserves an existing failure streak across %s probe data", (_label, probe) => { + const state = hasSeenHealthyKvProbe(); + bumpEscalation(stalled(), state, 3); + bumpEscalation(stalled(), state, 3); + const before = { ...state }; + expect(bumpEscalation(snap({ kvConnectivity: probe as HealthSnapshot["kvConnectivity"] }), state, 3)).toBe(false); + expect(state).toEqual(before); + expect(bumpEscalation(stalled(), state, 3)).toBe(true); + }); + it("escalates at most once per process", () => { const state = hasSeenHealthyKvProbe(); let fired = 0;