Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1804,7 +1804,14 @@ async function preflightCompressIfNeeded(
log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
return prepared;
}
return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false);
// #333: nothing left to fold is a STATIC state — failing fast here is
// a permanent 502 loop (every retry hits the same wall; the EMERGENCY
// nudge can never reach the model). Forward as-is: the upstream either
// accepts (its real window is larger than this limit — often a
// fallback/learned estimate) or rejects with a truthful 400 the client
// can act on (and the overflow self-heal learns the real window).
log("warn", `[${session.id}] preflight exhausted before starting: no part of the conversation is compressible (payload ~${payloadEstimate}/${limit}, model=${model}); forwarding the over-window payload as-is (#333)`);
return prepared;
}
log("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`);
// #300: stamp the chain marker so a downstream bili skips these
Expand Down Expand Up @@ -1843,13 +1850,31 @@ async function preflightCompressIfNeeded(
log("warn", `[${session.id}] preflight made no progress but the payload fits (~${result.payloadEstimate}/${limit}); forwarding as-is`);
return prepared;
}
// The payload still overflows the window: fail fast with a diagnostic
// error instead of forwarding a guaranteed-400 payload (#301).
// The payload still overflows the window.
const f = result.failure;
if (f?.kind === "aborted") {
log("warn", `[${session.id}] preflight aborted (${f.detail}); not forwarding`);
return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false };
}
if (f?.kind === "exhausted") {
// #333: exhaustion is a STATIC state — no retry can ever create a
// compressible range, so failing fast here is a permanent 502 loop.
// Forward the best payload we have (keeping any partial compression
// preflight already applied): the upstream either accepts (its real
// window is larger than this limit) and the injected nudge lets the
// model self-heal, or it rejects with a truthful 400 the client can
// act on (and the overflow self-heal learns the real window).
if (result.compressedRanges > 0) {
log("warn", `[${session.id}] preflight exhausted (${f.detail}) after ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved; forwarding the rebuilt over-window payload as-is (#333)`);
const rebuilt = runPrepare();
session.stats.requests -= 1;
return rebuilt;
}
log("warn", `[${session.id}] preflight exhausted (${f.detail}); forwarding the over-window payload as-is (#333)`);
return prepared;
}
// Transient upstream failure on the summarization call: fail fast with a
// diagnostic error instead of forwarding a guaranteed-400 payload (#301).
const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502;
return failFast(status, f?.detail ?? "unknown reason", status === 503);
}
Expand Down
90 changes: 74 additions & 16 deletions tests/preflight-fail-fast.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,23 @@ import { defaultConfig } from "acp-kernel";
import { startServer, type ProxyOptions } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";
import { listSessions } from "../src/session.ts";

// #301: when preflight (overflow) compression cannot proceed — the summary
// upstream is rate-limiting, or the compress budget is exhausted — the proxy
// must FAIL FAST with a structured error instead of forwarding the over-window
// payload as-is (guaranteed upstream 400, wasted quota, client retry storms;
// log evidence in #292). Forwarding as-is stays legal ONLY when the payload's
// own estimate actually fits the window (the trigger floors on
// stats.lastInputTokens, which can be stale — #300 double-counted usage).
// #301: when preflight (overflow) compression cannot proceed because the
// SUMMARY UPSTREAM failed (rate-limiting, error) — a transient state — the
// proxy must FAIL FAST with a structured error instead of forwarding the
// over-window payload as-is (guaranteed upstream 400, wasted quota, client
// retry storms; log evidence in #292).
// #333: when preflight cannot proceed because NOTHING IS LEFT TO FOLD (no
// compressible ranges — a static state no retry can change), failing fast is
// a permanent 502 loop: the client retries into the same wall forever and the
// EMERGENCY nudge can never reach the model. The proxy must forward the
// over-window payload as-is instead: the upstream either accepts (its real
// window is larger than bili's limit) or rejects with a truthful 400 the
// client can act on (and the overflow self-heal learns the real window).
// Forwarding as-is also stays legal when the payload's own estimate actually
// fits the window (the trigger floors on stats.lastInputTokens, which can be
// stale — #300 double-counted usage).

function okSse(inputTokens: number): string {
return (
Expand Down Expand Up @@ -190,7 +199,7 @@ test("e2e #301: payload fits the window + preflight 429 → request still forwar
}
});

test("e2e #301: over-window payload with nothing compressible → structured 502, no summary call, no forward", async () => {
test("e2e #333: over-window payload with nothing compressible → forwarded as-is (no 502 livelock)", async () => {
const calls: Call[] = [];
const upstream = makeUpstream429(calls);
upstream.listen(0, "127.0.0.1");
Expand All @@ -203,21 +212,70 @@ test("e2e #301: over-window payload with nothing compressible → structured 502

try {
// A single ~12k-token message against a 10k window: over-window, but
// the kernel never folds the lone (first) user message → no
// compressible ranges → preflight cannot proceed at all.
// the kernel never folds the lone (last) user message → no
// compressible ranges → preflight cannot proceed at all. #333: this
// is a STATIC state — failing fast here was a permanent 502 loop
// (every client retry hit the same wall). The proxy must forward the
// payload as-is and let the upstream decide.
const filler = "FILLER_".repeat(7000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-exhausted-sess" },
body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: filler }] }),
});
assert.equal(r.status, 502, "fail-fast 502 when nothing is compressible");
const json = JSON.parse(await r.text()) as { error?: { code?: string; message?: string; retryable?: boolean } };
assert.equal(json.error?.code, "preflight_compress_failed");
assert.equal(json.error?.retryable, false);
assert.ok(json.error?.message?.includes("NOT forwarded"), `message states the payload was withheld (got: ${json.error?.message})`);
assert.equal(r.status, 200, "#333: exhausted preflight forwards instead of looping on 502");
await r.text();
assert.equal(calls.filter((c) => !c.stream).length, 0, "no summarization call was spent on an incompressible payload");
assert.equal(calls.filter((c) => c.stream).length, 0, "the over-window payload was NOT forwarded upstream");
assert.equal(calls.filter((c) => c.stream).length, 1, "the over-window payload WAS forwarded upstream");
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});

test("e2e #333: over-window payload with nothing compressible → upstream 400 passed through + window self-heal", async () => {
const calls: Call[] = [];
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const raw = Buffer.concat(chunks).toString("utf8");
calls.push({ stream: true, body: raw });
res.writeHead(400, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: "This model's maximum context length is 8000 tokens", type: "invalid_request_error", code: "context_length_exceeded" } }));
});
});
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } });
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
// Same static state as above, but the upstream's REAL window is
// smaller than bili's configured one: the forwarded over-window
// payload is rejected. The client must see the upstream's truthful
// 400 (not a bili 502), and the overflow self-heal must learn the
// real window for the next turn.
const filler = "FILLER_".repeat(7000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-400-sess" },
body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: filler }] }),
});
assert.equal(r.status, 400, "the upstream's truthful 400 is passed through to the client");
const text = await r.text();
assert.ok(text.includes("maximum context length"), `client sees the upstream error (got: ${text})`);
assert.equal(calls.length, 1, "the over-window payload was forwarded upstream exactly once");

const sess = listSessions().find((s) => s.id === "preflight-400-sess");
assert.ok(sess, "session exists");
const learned = sess?.metadata.learnedContextLimits as Record<string, number> | undefined;
assert.equal(learned?.["claude-small"], 8000, "self-heal learned the real window from the 400");
} finally {
proxy.close();
await once(proxy, "close");
Expand Down
Loading