From f1076e5c03b0ad430815bfdc5877f5b1a9ca680c Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 29 Aug 2026 13:31:08 +0800 Subject: [PATCH] fix: forward over-window payload when preflight is exhausted (#333) --- src/server.ts | 31 +++++++++-- tests/preflight-fail-fast.test.ts | 90 +++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 19 deletions(-) diff --git a/src/server.ts b/src/server.ts index 60d2eac1a2..c2acd80a15 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1804,7 +1804,14 @@ async function preflightCompressIfNeeded( log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`); return prepared; } - return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false); + // #333: nothing left to fold is a STATIC state — failing fast here is + // a permanent 502 loop (every retry hits the same wall; the EMERGENCY + // nudge can never reach the model). Forward as-is: the upstream either + // accepts (its real window is larger than this limit — often a + // fallback/learned estimate) or rejects with a truthful 400 the client + // can act on (and the overflow self-heal learns the real window). + log("warn", `[${session.id}] preflight exhausted before starting: no part of the conversation is compressible (payload ~${payloadEstimate}/${limit}, model=${model}); forwarding the over-window payload as-is (#333)`); + return prepared; } log("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`); // #300: stamp the chain marker so a downstream bili skips these @@ -1843,13 +1850,31 @@ async function preflightCompressIfNeeded( log("warn", `[${session.id}] preflight made no progress but the payload fits (~${result.payloadEstimate}/${limit}); forwarding as-is`); return prepared; } - // The payload still overflows the window: fail fast with a diagnostic - // error instead of forwarding a guaranteed-400 payload (#301). + // The payload still overflows the window. const f = result.failure; if (f?.kind === "aborted") { log("warn", `[${session.id}] preflight aborted (${f.detail}); not forwarding`); return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false }; } + if (f?.kind === "exhausted") { + // #333: exhaustion is a STATIC state — no retry can ever create a + // compressible range, so failing fast here is a permanent 502 loop. + // Forward the best payload we have (keeping any partial compression + // preflight already applied): the upstream either accepts (its real + // window is larger than this limit) and the injected nudge lets the + // model self-heal, or it rejects with a truthful 400 the client can + // act on (and the overflow self-heal learns the real window). + if (result.compressedRanges > 0) { + log("warn", `[${session.id}] preflight exhausted (${f.detail}) after ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved; forwarding the rebuilt over-window payload as-is (#333)`); + const rebuilt = runPrepare(); + session.stats.requests -= 1; + return rebuilt; + } + log("warn", `[${session.id}] preflight exhausted (${f.detail}); forwarding the over-window payload as-is (#333)`); + return prepared; + } + // Transient upstream failure on the summarization call: fail fast with a + // diagnostic error instead of forwarding a guaranteed-400 payload (#301). const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502; return failFast(status, f?.detail ?? "unknown reason", status === 503); } diff --git a/tests/preflight-fail-fast.test.ts b/tests/preflight-fail-fast.test.ts index bc8b3867e2..92904c98ef 100644 --- a/tests/preflight-fail-fast.test.ts +++ b/tests/preflight-fail-fast.test.ts @@ -12,14 +12,23 @@ import { defaultConfig } from "acp-kernel"; import { startServer, type ProxyOptions } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; -// #301: when preflight (overflow) compression cannot proceed — the summary -// upstream is rate-limiting, or the compress budget is exhausted — the proxy -// must FAIL FAST with a structured error instead of forwarding the over-window -// payload as-is (guaranteed upstream 400, wasted quota, client retry storms; -// log evidence in #292). Forwarding as-is stays legal ONLY when the payload's -// own estimate actually fits the window (the trigger floors on -// stats.lastInputTokens, which can be stale — #300 double-counted usage). +// #301: when preflight (overflow) compression cannot proceed because the +// SUMMARY UPSTREAM failed (rate-limiting, error) — a transient state — the +// proxy must FAIL FAST with a structured error instead of forwarding the +// over-window payload as-is (guaranteed upstream 400, wasted quota, client +// retry storms; log evidence in #292). +// #333: when preflight cannot proceed because NOTHING IS LEFT TO FOLD (no +// compressible ranges — a static state no retry can change), failing fast is +// a permanent 502 loop: the client retries into the same wall forever and the +// EMERGENCY nudge can never reach the model. The proxy must forward the +// over-window payload as-is instead: the upstream either accepts (its real +// window is larger than bili's limit) or rejects with a truthful 400 the +// client can act on (and the overflow self-heal learns the real window). +// Forwarding as-is also stays legal when the payload's own estimate actually +// fits the window (the trigger floors on stats.lastInputTokens, which can be +// stale — #300 double-counted usage). function okSse(inputTokens: number): string { return ( @@ -190,7 +199,7 @@ test("e2e #301: payload fits the window + preflight 429 → request still forwar } }); -test("e2e #301: over-window payload with nothing compressible → structured 502, no summary call, no forward", async () => { +test("e2e #333: over-window payload with nothing compressible → forwarded as-is (no 502 livelock)", async () => { const calls: Call[] = []; const upstream = makeUpstream429(calls); upstream.listen(0, "127.0.0.1"); @@ -203,21 +212,70 @@ test("e2e #301: over-window payload with nothing compressible → structured 502 try { // A single ~12k-token message against a 10k window: over-window, but - // the kernel never folds the lone (first) user message → no - // compressible ranges → preflight cannot proceed at all. + // the kernel never folds the lone (last) user message → no + // compressible ranges → preflight cannot proceed at all. #333: this + // is a STATIC state — failing fast here was a permanent 502 loop + // (every client retry hit the same wall). The proxy must forward the + // payload as-is and let the upstream decide. const filler = "FILLER_".repeat(7000); const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-acp-session": "preflight-exhausted-sess" }, body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: filler }] }), }); - assert.equal(r.status, 502, "fail-fast 502 when nothing is compressible"); - const json = JSON.parse(await r.text()) as { error?: { code?: string; message?: string; retryable?: boolean } }; - assert.equal(json.error?.code, "preflight_compress_failed"); - assert.equal(json.error?.retryable, false); - assert.ok(json.error?.message?.includes("NOT forwarded"), `message states the payload was withheld (got: ${json.error?.message})`); + assert.equal(r.status, 200, "#333: exhausted preflight forwards instead of looping on 502"); + await r.text(); assert.equal(calls.filter((c) => !c.stream).length, 0, "no summarization call was spent on an incompressible payload"); - assert.equal(calls.filter((c) => c.stream).length, 0, "the over-window payload was NOT forwarded upstream"); + assert.equal(calls.filter((c) => c.stream).length, 1, "the over-window payload WAS forwarded upstream"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("e2e #333: over-window payload with nothing compressible → upstream 400 passed through + window self-heal", async () => { + const calls: Call[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + calls.push({ stream: true, body: raw }); + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "This model's maximum context length is 8000 tokens", type: "invalid_request_error", code: "context_length_exceeded" } })); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + // Same static state as above, but the upstream's REAL window is + // smaller than bili's configured one: the forwarded over-window + // payload is rejected. The client must see the upstream's truthful + // 400 (not a bili 502), and the overflow self-heal must learn the + // real window for the next turn. + const filler = "FILLER_".repeat(7000); + const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "preflight-400-sess" }, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: filler }] }), + }); + assert.equal(r.status, 400, "the upstream's truthful 400 is passed through to the client"); + const text = await r.text(); + assert.ok(text.includes("maximum context length"), `client sees the upstream error (got: ${text})`); + assert.equal(calls.length, 1, "the over-window payload was forwarded upstream exactly once"); + + const sess = listSessions().find((s) => s.id === "preflight-400-sess"); + assert.ok(sess, "session exists"); + const learned = sess?.metadata.learnedContextLimits as Record | undefined; + assert.equal(learned?.["claude-small"], 8000, "self-heal learned the real window from the 400"); } finally { proxy.close(); await once(proxy, "close");