From 814a2cdbdfa1e0c83c887b7ac7c0f33c11f66246 Mon Sep 17 00:00:00 2001 From: awork Date: Wed, 21 Oct 2026 02:27:52 +0800 Subject: [PATCH] fix: judge codex compaction gate against the request's own view (#4036) --- src/codex-compact.ts | 22 ++++++++++- src/server/handle.ts | 33 ++++++++++++++-- src/server/prepare-responses.ts | 19 ++++++++- tests/codex-compact-e2e.test.ts | 68 +++++++++++++++++++++++++++++++++ tests/codex-compact.test.ts | 10 +++++ 5 files changed, 144 insertions(+), 8 deletions(-) diff --git a/src/codex-compact.ts b/src/codex-compact.ts index 75ded778b5..96ce383e5e 100644 --- a/src/codex-compact.ts +++ b/src/codex-compact.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import type { CompressionBlock } from "acp-kernel"; +import { syncBlocks, type CompressionBlock, type CoreMessage } from "acp-kernel"; import { statusInputBaseline, type Session } from "./session.js"; import { codexCompactMode as knobCodexCompactMode } from "./knobs.js"; @@ -217,12 +217,30 @@ export function replaceBiliCompactionItems(input: T[]): { items: T[]; replace // line, not a model of codex's internal threshold. export const CODEX_COMPACT_HEALTH_RATIO = 0.9; -export function codexCompactGatePre(session: Session, effectiveLimit: number): boolean { +// #4036: `viewMessages` — when the caller passes the CURRENT request's core +// view, block activity is judged under THAT view via a pure syncBlocks probe +// instead of the stored active flags. The flags reflect the LAST synced view: +// a preceding short/side request legitimately deactivates every block whose +// carriers left its tiny view, and a compaction request arriving before any +// full-history recovery turn would read that stale deactivation and forward +// verbatim. The compact request itself carries the full history — its input +// is the authoritative evidence of whether the folds are still in play. +// syncBlocks deep-clones and never mutates its input, so the probe keeps the +// non-intercept path state-free (#335). Absent view → legacy stored-flag +// judgment. +export function codexCompactGatePre( + session: Session, + effectiveLimit: number, + viewMessages?: CoreMessage[], +): boolean { if (effectiveLimit <= 0) return false; // Evidence-grade baseline only (#3008/#1492/#1839 discipline): raw // stats.lastInputTokens can be an uncalibrated estimate-grade value that // self-disables the valve at phantom pressure while real usage is far below. if (statusInputBaseline(session) >= effectiveLimit * CODEX_COMPACT_HEALTH_RATIO) return false; + if (viewMessages !== undefined) { + return syncBlocks(viewMessages, session.state).state.blocks.some((b) => b.active); + } return session.state.blocks.some((b) => b.active); } diff --git a/src/server/handle.ts b/src/server/handle.ts index 7f19954fe3..31ec68b700 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -5,8 +5,9 @@ import { Readable } from "node:stream"; import fs from "node:fs"; import path from "node:path"; import { performance } from "node:perf_hooks"; -import { defaultPrompts, type CompressionCore, type Config, type PackSurface, type Prompts } from "acp-kernel"; +import { defaultPrompts, syncBlocks, type CompressionCore, type Config, type CoreMessage, type PackSurface, type Prompts } from "acp-kernel"; import { conversationSignalAnthropic, conversationSignalGoogle, conversationSignalOpenai, conversationIdentityResponses, conversationSignalResponses, googleSystemText, type AnthropicRequestBody, type GoogleRequestBody, type OpenAIRequestBody, type ResponsesRequestBody } from "acp-kernel/wire"; +import { responsesToCoreWithToolImages as responsesToCore } from "../responses-tool-output.js"; import { resolveCompress, resolveCompressPrompts, resolveCompressSurfaceDetailed, resolveRequestConfig } from "../compress-settings.js"; import { FALLBACK_EFFECTIVE_WINDOW_FLOOR, findRoute, findRouteKey, lookupContextLimit, resolveConfiguredContextLimit, resolveConfiguredOutputLimit, resolveCompressProtocol, resolveDeclaredProtocol, type ProxyOptions } from "../config.js"; import { resolveProxyDecision } from "../upstream-proxy.js"; @@ -17,7 +18,7 @@ import { hostIdForLog, maskHeadersForLog, maskUrlForLog, maskUrlsInText } from " import { bodyMightCarryImages, buildIncomingImageIndex, pruneRetrieveImgExports, refreshIncomingImageIndex, type IndexedImage } from "../image-restore.js"; import { durableMessageToolPaths, withDurableProtection } from "../durable-message-guards.js"; import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, pluginLaneDegraded, pluginLaneRestore } from "../session-self-heal.js"; -import { acquireInFlight, getSession, hasProcessedState, markDirty, peekSession, releaseInFlight, storeEffectiveConfig, tickPostRebuildAnchor, withSessionLock, type Session } from "../session.js"; +import { acquireInFlight, getSession, hasProcessedState, markDirty, peekSession, releaseInFlight, statusInputBaseline, storeEffectiveConfig, tickPostRebuildAnchor, withSessionLock, type Session } from "../session.js"; import { buildCompressSystemPrompt } from "../compress-tool.js"; import { storeEffectiveSearchPlanAware } from "../decompress-shared.js"; import { externalSummaryEnabled } from "../external-summary-surface.js"; @@ -2351,7 +2352,19 @@ async function handleRequest( hasCompactionTrigger((parsed as ResponsesRequestBody).input); if (isCodexCompactTrigger) { const mode = codexCompactMode(); - const gatePre = codexCompactGatePre(session, reqConfig.modelContextLimit); + // #4036: judge block activity under THIS trigger's own view + // (it carries the full history), not the last-synced stored + // flags — a preceding short-view request legitimately + // deactivated the blocks and would otherwise steer this + // trigger verbatim to an upstream that 400s on it. + // Unparseable view → legacy stored-flag judgment. + let gateView: CoreMessage[] | undefined; + try { + gateView = responsesToCore(parsed as ResponsesRequestBody).msgs; + } catch { + gateView = undefined; + } + const gatePre = codexCompactGatePre(session, reqConfig.modelContextLimit, gateView); if (mode === "intercept" && gatePre) prepared = await runPrepare(); if (prepared?.codexForge) { logRequestCost(log, session.id, inboundMsgs, inboundBytes, reqT0, prepared.body); @@ -2365,7 +2378,19 @@ async function handleRequest( const normalized = replaced + dropped > 0; const forwardBody = normalized ? Buffer.from(JSON.stringify({ ...original, input: items })) : bodyBuffer; const why = mode !== "intercept" ? "BILI_CODEX_COMPACT=pass" : !gatePre ? "gate preconditions not met" : "transform/forge failed"; - log("info", `[${session.id}] codex compaction_trigger request not intercepted (${why}) — forwarding ${normalized ? `with bili summaries normalized (replaced=${replaced}, dropped=${dropped})` : "verbatim"} (no preflight, no rebuild, no window clamp)`); + // #4036 observability: the gate's own inputs at decision time — + // limit, evidence-grade baseline + source, active/total blocks + // under the view actually judged, and this request's inbound + // message count (the prior request's view cardinality is the + // preceding "processTurn: N msgs" line in the tee log). + let gateDetail = ""; + if (mode === "intercept" && !gatePre) { + const activeCount = gateView !== undefined + ? syncBlocks(gateView, session.state).state.blocks.filter((b) => b.active).length + : session.state.blocks.filter((b) => b.active).length; + gateDetail = ` [limit=${reqConfig.modelContextLimit} baseline=${statusInputBaseline(session)}(${session.stats.lastInputTokensSource ?? "none"}) active=${activeCount}/${session.state.blocks.length}${gateView !== undefined ? "@view" : "@stored"} inboundMsgs=${inboundMsgs}]`; + } + log("info", `[${session.id}] codex compaction_trigger request not intercepted (${why})${gateDetail} — forwarding ${normalized ? `with bili summaries normalized (replaced=${replaced}, dropped=${dropped})` : "verbatim"} (no preflight, no rebuild, no window clamp)`); logRequestCost(log, session.id, inboundMsgs, inboundBytes, reqT0, forwardBody); return { body: forwardBody, prepared: null }; } diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 1bbd183c6e..8067dc1bfd 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -726,8 +726,20 @@ export function prepareResponsesCompact( // request passes through verbatim without running processTurn (no state // mutation as a side effect of a compact that will not be intercepted). // #3686: host policy, not wire profile — declared plugin hosts are exempt. + // #4036: judge block activity under THIS compact request's own view (the + // pure probe below) instead of the last-synced stored flags — the same + // stale-deactivation window as the trigger form. The projection is reused + // by the transform so the body converts once, not twice. + let gateProjection: ReturnType | undefined; + if (Array.isArray(parsed.input)) { + try { + gateProjection = responsesToCore(forgeBody); + } catch { + gateProjection = undefined; + } + } if (codexCompactMode() !== "intercept" || !isCodexHost(req.headers) || !Array.isArray(parsed.input) - || !codexCompactGatePre(session, config.modelContextLimit)) { + || !codexCompactGatePre(session, config.modelContextLimit, gateProjection?.msgs)) { return base; } // The state commit below is all-or-nothing: every non-forge path restores @@ -743,7 +755,10 @@ export function prepareResponsesCompact( // foreign compaction blob. let transformOk = false; try { - const projection = responsesToCore(forgeBody); + // gateProjection is undefined only when the pre-gate probe threw on an + // unparseable body — re-convert here and fail into the same + // restore-and-passthrough catch as before (#335). + const projection = gateProjection ?? responsesToCore(forgeBody); // The forged handoff is one-shot with no tool channel: strip absorb so // no [ACP absorb] instruction bakes into the forged history, and run // the absorb view so absorbed pairs stay hidden in it (wire parity). diff --git a/tests/codex-compact-e2e.test.ts b/tests/codex-compact-e2e.test.ts index c5762929b0..21012a8215 100644 --- a/tests/codex-compact-e2e.test.ts +++ b/tests/codex-compact-e2e.test.ts @@ -504,6 +504,74 @@ test("e2e #332 (trigger form): intercept + healthy ACP + oversized payload → f }); }); +// Issue #4036: a preceding short/side request legitimately deactivates every +// block (its tiny view carries none of their carriers — reap needs 3 passes, +// so one short view only flips active=false), and the trigger that follows +// IMMEDIATELY — before any full-history recovery turn — must be judged against +// its OWN view (the full history) instead of the stale stored active flags. +// Stale flags → gate false → the oversized trigger forwarded verbatim → the +// field event's upstream 400. + +test("e2e #4036 (trigger form): short side request deactivates blocks → immediate full-history trigger still intercepted", async () => { + await withHarness({ mode: "intercept", firstTurnTokens: 1000 }, async (h) => { + const afterSetup = await setupCompressedSession(h); + // Field-equivalent short request on the SAME session, no recovery turn: + // its one-message view carries no block carrier → syncBlocks deactivates all. + const rs = await fetch(h.url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: SESSION, instructions: "You are the test coding agent.", input: [{ type: "message", role: "user", content: "Create a title" }] }), + }); + assert.equal(rs.status, 200); + await rs.text(); + const sDeact = listSessions().find((x) => x.meta.label === SESSION)!; + assert.ok(sDeact.state.blocks.length > 0, "blocks retained after the short view"); + assert.ok(sDeact.state.blocks.every((b) => !b.active), "short view deactivated every block"); + + // Immediate trigger carrying the FULL history — before any recovery turn. + const r2 = await fetch(h.url, { + method: "POST", + headers: { "content-type": "application/json", "user-agent": CODEX_UA }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: SESSION, instructions: "You are the test coding agent.", input: [...conversation(), { type: "compaction_trigger" }] }), + }); + assert.equal(r2.status, 200); + assert.equal(r2.headers.get("content-type"), "text/event-stream", "forged SSE"); + const text = await r2.text(); + assert.ok(text.includes("fc_bili_"), "forged compaction item despite stale deactivation"); + assert.ok(text.includes("resp_bili_"), "forged response id"); + assert.equal(h.bodies.length, afterSetup + 1, "trigger never reached upstream"); + const sAfter = listSessions().find((x) => x.meta.label === SESSION)!; + assert.ok(sAfter.state.blocks.some((b) => b.active), "the trigger's own view re-activated the fold"); + }); +}); + +test("e2e #4036 (negative): trigger whose OWN view carries no fold is not intercepted and mutates no state", async () => { + await withHarness({ mode: "intercept", firstTurnTokens: 1000 }, async (h) => { + const afterSetup = await setupCompressedSession(h); + const s = listSessions().find((x) => x.meta.label === SESSION)!; + assert.ok(s.state.blocks.some((b) => b.active), "stored flags say active pre-trigger"); + const stateBefore = JSON.stringify(s.state); + // Fresh unrelated history + trigger: the folded messages are absent from + // this view, so forging would hand off a summary the client's payload + // does not contain — the gate must stay honest and forward verbatim. + const fresh = Array.from({ length: 3 }, (_, i) => ({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Fresh unrelated history ${i}. ` + `FRESH_${i}_content_`.repeat(100) })); + const rawBody = JSON.stringify({ model: "gpt-resp", stream: true, session_id: SESSION, instructions: "You are the test coding agent.", input: [...fresh, { type: "compaction_trigger" }] }); + const r = await fetch(h.url, { + method: "POST", + headers: { "content-type": "application/json", "user-agent": CODEX_UA }, + body: rawBody, + }); + assert.equal(r.status, 200); + await r.text(); + assert.equal(h.bodies.length, afterSetup + 1, "forwarded to upstream, not forged"); + assert.equal(h.bodies[h.bodies.length - 1], rawBody, "upstream received the exact bytes codex sent"); + const fwd = JSON.parse(h.bodies[afterSetup]) as { input: { type: string }[] }; + assert.equal(fwd.input[fwd.input.length - 1].type, "compaction_trigger", "trigger stays final"); + const after = listSessions().find((x) => x.meta.label === SESSION)!; + assert.equal(JSON.stringify(after.state), stateBefore, "no prepare ran on the rejected trigger (state untouched)"); + }); +}); + test("e2e #332 (endpoint form): intercept + gate preconditions fail → raw body forwarded, no state mutation", async () => { await withHarness({ mode: "intercept", firstTurnTokens: 15000 }, async (h) => { const afterSetup = await setupCompressedSession(h); diff --git a/tests/codex-compact.test.ts b/tests/codex-compact.test.ts index ddf350218d..c05ad3f485 100644 --- a/tests/codex-compact.test.ts +++ b/tests/codex-compact.test.ts @@ -142,6 +142,16 @@ test("codexCompactGatePre: evidence-grade baseline only (#3008)", () => { assert.equal(codexCompactGatePre(mockSession(9_500, [block(true)]), limit), true, "no source, no usage anchor → baseline 0 → forge"); }); +test("codexCompactGatePre: view-aware block activity (#4036)", () => { + const limit = 10_000; + const foldBlock = (active: boolean): CompressionBlock => ({ ...block(active), effectiveMessageIds: ["h_fold_raw"] }); + const viewMsg = (id: string) => ({ id, role: "user" as const, contentType: "text" as const, text: "x" }); + assert.equal(codexCompactGatePre(mockSession(1_000, [foldBlock(false)]), limit, [viewMsg("h_fold_raw")]), true, "stored-inactive (prior short view) but carrier present in THIS view → forge"); + assert.equal(codexCompactGatePre(mockSession(1_000, [foldBlock(true)]), limit, [viewMsg("h_unrelated")]), false, "stored-active but carrier absent from THIS view → honest pass-through"); + assert.equal(codexCompactGatePre(mockSession(1_000, [foldBlock(false)]), limit), false, "no view argument → legacy stored-flag judgment"); + assert.equal(codexCompactGatePre(mockSession(9_500, [foldBlock(false)], { source: "usage" }), limit, [viewMsg("h_fold_raw")]), false, "view does not loosen the 90% baseline wall"); +}); + test("replaceBiliCompactionItems: echo becomes a summary handoff message in place", () => { const ours = { type: "compaction", id: `${CODEX_COMPACT_ID_PREFIX}x`, encrypted_content: `${CODEX_COMPACT_SENTINEL}summary text` }; const real = { type: "compaction", id: "fc_real", encrypted_content: "opaque" };