diff --git a/src/preflight.ts b/src/preflight.ts index 5390cacbc3..302892578d 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -224,6 +224,27 @@ async function summarizeRange(deps: PreflightDeps, content: string, startRef: st const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client disconnected during preflight compression" }; +// #330: the soft-protected recent zone (preserveRecentMessages / +// preserveRecentTokens / most-recent user message) can cover ALL foldable +// content when one large recent message pushes the payload over a small +// window — preflight then 502s forever with no recovery path. Under overflow +// the soft zone is a preference, not a constraint: relax it to zero so its +// oldest content becomes foldable. The hard protectedTools exclusion is +// computed independently of preserveRecent* and still applies. +function relaxedConfig(config: Config): Config { + return { ...config, preserveRecentMessages: 0, preserveRecentTokens: 0 }; +} + +// #330: the preflight's fit check must reflect the durable payload, not the +// per-turn emergency-truncate side effect — that node would trim the very tool +// output overflowing the window, making currentTokens undershoot and the loop +// break before the soft zone is relaxed. Inflate the window so usage stays +// below truncate.threshold; compressible ranges derive from the protected zone, +// not usage, so this is safe. +function noEmergencyTruncate(config: Config): Config { + return { ...config, modelContextLimit: config.modelContextLimit * 100 }; +} + export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise { const limit = deps.config.modelContextLimit; const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) }; @@ -239,6 +260,12 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa let currentTokens = deps.session.stats.lastInputTokens; let startTokens = -1; let failure: PreflightFailure | undefined; + let activeConfig = deps.config; + let relaxed = false; + const relaxedExhaustedDetail = + `the payload still exceeds the window after folding everything compressible, including the soft-protected recent zone ` + + `(last ${deps.config.preserveRecentMessages} messages + most recent user message), which was relaxed under overflow; hard protectedTools remain excluded. ` + + `Raise the model context window or restart the session to recover.`; for (let round = 0; round < MAX_PREFLIGHT_ROUNDS; round++) { if (deps.signal?.aborted) { failure = ABORTED_FAILURE; @@ -249,7 +276,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const turn = deps.core.processTurn({ messages, state: deps.session.state, - config: deps.config, + config: noEmergencyTruncate(activeConfig), tokenCount: currentTokens, renderTags: "text-only", }); @@ -265,7 +292,19 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (currentTokens < limit) break; const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []); if (ranges.length === 0) { - failure = { kind: "exhausted", detail: "no compressible ranges remain in the conversation" }; + // #330: nothing foldable outside the soft-protected recent zone. + // Relax the soft zone (oldest-first within it) and retry — the hard + // protectedTools exclusion still applies. Gate on the payload's own + // estimate (not currentTokens, which is floored by a possibly-stale + // lastInputTokens from a prior model): if the real payload already + // fits, stop instead of folding protected content. + if (!relaxed && result.payloadEstimate >= limit) { + activeConfig = relaxedConfig(deps.config); + relaxed = true; + deps.log("warn", "[preflight] no compressible ranges outside the protected recent zone; relaxing soft protection (preserveRecentMessages/Tokens -> 0) and retrying"); + continue; + } + failure = { kind: "exhausted", detail: relaxed ? relaxedExhaustedDetail : "no compressible ranges remain in the conversation" }; break; } const range = [...ranges].sort((a, b) => refNum(a.startRef) - refNum(b.startRef))[0]; @@ -315,7 +354,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (!summary) continue; const ctx: RewriteCtx = { core: deps.core, - config: deps.config, + config: activeConfig, messages, session: deps.session, log: (msg) => deps.log("info", msg), @@ -342,7 +381,12 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa } } if (!failure && currentTokens >= limit) { - failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds` }; + failure = { + kind: "exhausted", + detail: relaxed + ? relaxedExhaustedDetail + : `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds`, + }; } if (result.compressedRanges > 0) deps.session.stats.lastInputTokens = currentTokens; result.savedTokens = Math.max(0, startTokens - currentTokens); diff --git a/src/server.ts b/src/server.ts index a63362af16..1d1e19c518 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2564,13 +2564,19 @@ async function preflightCompressIfNeeded( log("error", `[${session.id}] preflight fail-fast ${status} (retryable=${retryable}): ${message}`); return { failFast: true, status, message, retryable, respond: !res.writableEnded }; }; - if ((prepared.nudge?.compressibleRanges ?? []).length === 0) { - if (payloadEstimate < limit) { - log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`); - return prepared; - } - return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false); + if ((prepared.nudge?.compressibleRanges ?? []).length === 0 && payloadEstimate < limit) { + // #300: the trigger fired on a stale baseline (lastInputTokens) but the + // payload's own estimate fits the window — forwarding as-is is safe. + log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`); + return prepared; } + // #330: the payload overflows the window (or nothing is foldable in the + // normal pass but it doesn't fit). Let preflightCompress try to fold it — + // it relaxes the soft-protected recent zone when nothing is foldable + // outside it, and fails fast with an actionable error only when truly + // nothing is foldable (no summarization call is spent in that case). The + // old pre-check failed fast here on the normal-config compressibleRanges, + // which excluded the soft zone — bricking the #330 livelock. log("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`); // #300: stamp the chain marker so a downstream bili skips these // summarization calls too (preflight always processes). @@ -2597,16 +2603,21 @@ async function preflightCompressIfNeeded( }, prepared.originalMessages, ); - if (result.payloadEstimate < limit) { - if (result.compressedRanges > 0) { - log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`); - const rebuilt = runPrepare(); - // runPrepare re-incremented stats.requests; the rebuild is internal - // to this single client request. - session.stats.requests -= 1; - return rebuilt; - } - log("warn", `[${session.id}] preflight made no progress but the payload fits (~${result.payloadEstimate}/${limit}); forwarding as-is`); + // #330: decide forward/fail on the payload actually forwarded, not + // result.payloadEstimate — the preflight's relaxed-zone processTurn trims + // that estimate more than the normal-config prepare does, which can turn a + // guaranteed-400 forward into a false "fits". Images ride the payload + // verbatim (#488): add their cost back or #496's image-dominated payload + // would look "fitting" on its text estimate alone. + if (result.compressedRanges > 0) { + log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`); + const rebuilt = runPrepare(); + // runPrepare re-incremented stats.requests; the rebuild is internal + // to this single client request. + session.stats.requests -= 1; + if (estimateCoreMessages(rebuilt.processedMessages) + imageTokens < limit) return rebuilt; + } else if (estimateCoreMessages(prepared.processedMessages) + imageTokens < limit) { + log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`); return prepared; } // The payload still overflows the window: fail fast with a diagnostic @@ -2617,7 +2628,7 @@ async function preflightCompressIfNeeded( return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false }; } const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502; - return failFast(status, f?.detail ?? "unknown reason", status === 503); + return failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", status === 503); } async function forward( diff --git a/tests/preflight-fail-fast.test.ts b/tests/preflight-fail-fast.test.ts index bc8b3867e2..45b524859e 100644 --- a/tests/preflight-fail-fast.test.ts +++ b/tests/preflight-fail-fast.test.ts @@ -8,7 +8,7 @@ process.env.NODE_ENV = "test"; // exponential backoff — these tests are about the post-retry behavior. process.env.BILI_REPLAY_RETRY_MAX = "1"; -import { defaultConfig } from "acp-kernel"; +import { defaultConfig, type Config } from "acp-kernel"; import { startServer, type ProxyOptions } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; @@ -64,7 +64,30 @@ function makeUpstream429(calls?: Call[]): http.Server { }); } -function startProxy(upstreamPort: number, models: Record): Promise { +// Upstream that succeeds on BOTH the forwarded (stream) request and the +// preflight summarization (non-stream) call — needed to exercise the #330 +// relaxed-zone fold path end to end. +function makeUpstreamOk(calls?: Call[]): http.Server { + return http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls?.push({ stream: !!parsed.stream, body: raw }); + if (parsed.stream) { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse(1000)); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ content: [{ type: "text", text: "SUMMARY: the large recent user message was a deterministic load-growth payload; its raw content is no longer needed." }] })); + } + }); + }); +} + +function startProxy(upstreamPort: number, models: Record, kernelOverrides?: Partial): Promise { _setStoreForTest(new SessionStore({ enabled: false })); setRegistryForTest({}); return startServer({ @@ -73,7 +96,7 @@ function startProxy(upstreamPort: number, models: Record !c.stream).length, 0, "no summarization call was spent on an incompressible payload"); assert.equal(calls.filter((c) => c.stream).length, 0, "the over-window payload was NOT forwarded upstream"); } finally { @@ -225,3 +259,48 @@ test("e2e #301: over-window payload with nothing compressible → structured 502 await once(upstream, "close"); } }); + +test("e2e #330: over-window payload whose only foldable content is in the protected recent zone → soft zone relaxed, folded, forwarded", async () => { + const calls: Call[] = []; + const upstream = makeUpstreamOk(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + // Three messages against a 10k window: a small opener plus two large + // recent messages (~8k tokens each, ~16k total). All three sit inside + // the soft-protected recent zone (preserveRecentMessages=5 covers all + // three), so the normal pass finds nothing foldable. Before #330 this + // 502'd forever; now preflight relaxes the soft zone, folds the oldest + // large message, and forwards the now-fitting payload. + const big1 = "A".repeat(32000); + const big2 = "B".repeat(32000); + const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "preflight-relax-sess" }, + body: JSON.stringify({ + model: "claude-small", + max_tokens: 1024, + stream: true, + messages: [ + { role: "user", content: "start the task" }, + { role: "assistant", content: big1 }, + { role: "user", content: big2 }, + ], + }), + }); + assert.equal(r.status, 200, "the over-window payload is folded and forwarded, not 502'd"); + assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made the summarization call to fold the protected message"); + assert.equal(calls.filter((c) => c.stream).length, 1, "the folded payload was forwarded upstream"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +});