From e32d66c3c010bcff10fcd91853a0db77f22f8bbb Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 17 Oct 2026 18:31:54 +0800 Subject: [PATCH 1/2] debug(#3386): staged heap probes over the local processing window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One oversized request aborting the whole process (#3386: 25 MiB / 1372 msgs → 4 GB heap) cannot be attributed to its allocating stage from outside. Three mem-probe samples, gated on the existing --debug flag (zero new config surface, zero behavior change when off): - inbound-parsed: after readBody+decode+toString+JSON.parse and session identity resolution — baseline before the heavy local window work - prepared: after runPrepare returns (both the codex-intercept and the main path) — cumulative cost of image-index build, wire→core conversion, kernel processTurn, and outbound rebuild/stringify Each line logs heapUsed/external/rss plus the stage's payload bytes, prefixed 'mem-probe :' for grep continuity. The delta between consecutive samples is what tells the owner whether the OOM mass sits in parse materialization or in the fold/processTurn window — input for the #3386 A/B/C fix decision. --- src/server/handle.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/server/handle.ts b/src/server/handle.ts index de34135e3f..414599e520 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -223,6 +223,19 @@ export async function handle( } } +// #3386: staged heap probes over the local processing window. One oversized +// request can abort the whole process mid-turn (#3386: 25 MiB / 1372 msgs → +// 4 GB heap), and attributing WHICH stage materialized it (raw buffer vs +// toString+JSON.parse graph vs kernel/processTurn rebuild) needs a heap +// sample at each boundary. Gated on the existing --debug flag: zero new +// config surface, zero behavior change when off. +function memProbe(debug: boolean, log: (level: string, msg: string) => void, sessionId: string, stage: string, payloadBytes?: number): void { + if (!debug) return; + const mu = process.memoryUsage(); + const mi = (n: number): string => `${Math.round(n / 1048576)}MiB`; + log("debug", `[${sessionId}] mem-probe ${stage}: heap=${mi(mu.heapUsed)} external=${mi(mu.external)} rss=${mi(mu.rss)}${payloadBytes !== undefined ? ` payload=${payloadBytes}B` : ""}`); +} + async function handleRequest( req: http.IncomingMessage, res: http.ServerResponse, @@ -1323,6 +1336,7 @@ async function handleRequest( ? bodyIdentity.value : clientConversationHeader(req.headers); const session = getSession(sessionId, { protocol, upstreamOrigin, label: clientLabel ?? (anonAffinity ? `prefix-affinity ${anonAffinity.tailHash.slice(0, 8)}` : undefined) }); + memProbe(opts.debug, log, sessionId, "inbound-parsed", bodyBuffer.length); if (designNamespaced && session.metadata.personaNamespace !== true) session.metadata.personaNamespace = true; let publicForkPrefix = false; if (!countTokens && !responsesCompact && session.metadata.publicForkReceipt !== undefined) { @@ -2294,6 +2308,7 @@ async function handleRequest( const mode = codexCompactMode(); const gatePre = codexCompactGatePre(session, reqConfig.modelContextLimit); if (mode === "intercept" && gatePre) prepared = await runPrepare(); + if (prepared) memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? prepared.body.length : prepared.body.byteLength); if (prepared?.codexForge) { logRequestCost(log, session.id, inboundMsgs, inboundBytes, reqT0, prepared.body); return { body: prepared.body, prepared }; @@ -2311,6 +2326,7 @@ async function handleRequest( return { body: forwardBody, prepared: null }; } prepared = await runPrepare(); + memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? prepared.body.length : prepared.body.byteLength); // [#2926] anthropic Threads lane: prepare mirrored the turn and // may have armed a reforge (mirror estimate crossed the window // on a continue, rollout flag on). Execute it now — one summary From 982ecf240e680722e933baea2fe7bef19f21a379 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 17 Oct 2026 20:20:39 +0800 Subject: [PATCH 2/2] fix(#3386): mem-probe payload field reports UTF-8 bytes for string bodies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review nit on the staged heap probes: the log line contract says payload=B (bytes) and inbound-parsed samples bodyBuffer.length (bytes), but the prepared-stage sampled prepared.body.length — UTF-16 code units for string bodies. Identical for ASCII/base64 payloads, under-reports multibyte text, and makes the inter-sample payload delta (unit-mixed) misleading exactly when comparing stages. Buffer.byteLength is allocation-free, so the debug path stays cheap. --- src/server/handle.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/handle.ts b/src/server/handle.ts index 414599e520..8295b6703b 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -2308,7 +2308,7 @@ async function handleRequest( const mode = codexCompactMode(); const gatePre = codexCompactGatePre(session, reqConfig.modelContextLimit); if (mode === "intercept" && gatePre) prepared = await runPrepare(); - if (prepared) memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? prepared.body.length : prepared.body.byteLength); + if (prepared) memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? Buffer.byteLength(prepared.body) : prepared.body.byteLength); if (prepared?.codexForge) { logRequestCost(log, session.id, inboundMsgs, inboundBytes, reqT0, prepared.body); return { body: prepared.body, prepared }; @@ -2326,7 +2326,7 @@ async function handleRequest( return { body: forwardBody, prepared: null }; } prepared = await runPrepare(); - memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? prepared.body.length : prepared.body.byteLength); + memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? Buffer.byteLength(prepared.body) : prepared.body.byteLength); // [#2926] anthropic Threads lane: prepare mirrored the turn and // may have armed a reforge (mirror estimate crossed the window // on a continue, rollout flag on). Execute it now — one summary