Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,27 @@ async function summarizeRange(deps: PreflightDeps, content: string, startRef: st

const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client disconnected during preflight compression" };

// #330: the soft-protected recent zone (preserveRecentMessages /
// preserveRecentTokens / most-recent user message) can cover ALL foldable
// content when one large recent message pushes the payload over a small
// window — preflight then 502s forever with no recovery path. Under overflow
// the soft zone is a preference, not a constraint: relax it to zero so its
// oldest content becomes foldable. The hard protectedTools exclusion is
// computed independently of preserveRecent* and still applies.
function relaxedConfig(config: Config): Config {
return { ...config, preserveRecentMessages: 0, preserveRecentTokens: 0 };
}

// #330: the preflight's fit check must reflect the durable payload, not the
// per-turn emergency-truncate side effect — that node would trim the very tool
// output overflowing the window, making currentTokens undershoot and the loop
// break before the soft zone is relaxed. Inflate the window so usage stays
// below truncate.threshold; compressible ranges derive from the protected zone,
// not usage, so this is safe.
function noEmergencyTruncate(config: Config): Config {
return { ...config, modelContextLimit: config.modelContextLimit * 100 };
}

export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise<PreflightResult> {
const limit = deps.config.modelContextLimit;
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
Expand All @@ -239,6 +260,12 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
let currentTokens = deps.session.stats.lastInputTokens;
let startTokens = -1;
let failure: PreflightFailure | undefined;
let activeConfig = deps.config;
let relaxed = false;
const relaxedExhaustedDetail =
`the payload still exceeds the window after folding everything compressible, including the soft-protected recent zone ` +
`(last ${deps.config.preserveRecentMessages} messages + most recent user message), which was relaxed under overflow; hard protectedTools remain excluded. ` +
`Raise the model context window or restart the session to recover.`;
for (let round = 0; round < MAX_PREFLIGHT_ROUNDS; round++) {
if (deps.signal?.aborted) {
failure = ABORTED_FAILURE;
Expand All @@ -249,7 +276,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
const turn = deps.core.processTurn({
messages,
state: deps.session.state,
config: deps.config,
config: noEmergencyTruncate(activeConfig),
tokenCount: currentTokens,
renderTags: "text-only",
});
Expand All @@ -265,7 +292,19 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
if (currentTokens < limit) break;
const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
if (ranges.length === 0) {
failure = { kind: "exhausted", detail: "no compressible ranges remain in the conversation" };
// #330: nothing foldable outside the soft-protected recent zone.
// Relax the soft zone (oldest-first within it) and retry — the hard
// protectedTools exclusion still applies. Gate on the payload's own
// estimate (not currentTokens, which is floored by a possibly-stale
// lastInputTokens from a prior model): if the real payload already
// fits, stop instead of folding protected content.
if (!relaxed && result.payloadEstimate >= limit) {
activeConfig = relaxedConfig(deps.config);
relaxed = true;
deps.log("warn", "[preflight] no compressible ranges outside the protected recent zone; relaxing soft protection (preserveRecentMessages/Tokens -> 0) and retrying");
continue;
}
failure = { kind: "exhausted", detail: relaxed ? relaxedExhaustedDetail : "no compressible ranges remain in the conversation" };
break;
}
const range = [...ranges].sort((a, b) => refNum(a.startRef) - refNum(b.startRef))[0];
Expand Down Expand Up @@ -315,7 +354,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
if (!summary) continue;
const ctx: RewriteCtx = {
core: deps.core,
config: deps.config,
config: activeConfig,
messages,
session: deps.session,
log: (msg) => deps.log("info", msg),
Expand All @@ -342,7 +381,12 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
}
}
if (!failure && currentTokens >= limit) {
failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds` };
failure = {
kind: "exhausted",
detail: relaxed
? relaxedExhaustedDetail
: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds`,
};
}
if (result.compressedRanges > 0) deps.session.stats.lastInputTokens = currentTokens;
result.savedTokens = Math.max(0, startTokens - currentTokens);
Expand Down
45 changes: 28 additions & 17 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2564,13 +2564,19 @@ async function preflightCompressIfNeeded(
log("error", `[${session.id}] preflight fail-fast ${status} (retryable=${retryable}): ${message}`);
return { failFast: true, status, message, retryable, respond: !res.writableEnded };
};
if ((prepared.nudge?.compressibleRanges ?? []).length === 0) {
if (payloadEstimate < limit) {
log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
return prepared;
}
return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false);
if ((prepared.nudge?.compressibleRanges ?? []).length === 0 && payloadEstimate < limit) {
// #300: the trigger fired on a stale baseline (lastInputTokens) but the
// payload's own estimate fits the window — forwarding as-is is safe.
log("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
return prepared;
}
// #330: the payload overflows the window (or nothing is foldable in the
// normal pass but it doesn't fit). Let preflightCompress try to fold it —
// it relaxes the soft-protected recent zone when nothing is foldable
// outside it, and fails fast with an actionable error only when truly
// nothing is foldable (no summarization call is spent in that case). The
// old pre-check failed fast here on the normal-config compressibleRanges,
// which excluded the soft zone — bricking the #330 livelock.
log("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`);
// #300: stamp the chain marker so a downstream bili skips these
// summarization calls too (preflight always processes).
Expand All @@ -2597,16 +2603,21 @@ async function preflightCompressIfNeeded(
},
prepared.originalMessages,
);
if (result.payloadEstimate < limit) {
if (result.compressedRanges > 0) {
log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`);
const rebuilt = runPrepare();
// runPrepare re-incremented stats.requests; the rebuild is internal
// to this single client request.
session.stats.requests -= 1;
return rebuilt;
}
log("warn", `[${session.id}] preflight made no progress but the payload fits (~${result.payloadEstimate}/${limit}); forwarding as-is`);
// #330: decide forward/fail on the payload actually forwarded, not
// result.payloadEstimate — the preflight's relaxed-zone processTurn trims
// that estimate more than the normal-config prepare does, which can turn a
// guaranteed-400 forward into a false "fits". Images ride the payload
// verbatim (#488): add their cost back or #496's image-dominated payload
// would look "fitting" on its text estimate alone.
if (result.compressedRanges > 0) {
log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`);
const rebuilt = runPrepare();
// runPrepare re-incremented stats.requests; the rebuild is internal
// to this single client request.
session.stats.requests -= 1;
if (estimateCoreMessages(rebuilt.processedMessages) + imageTokens < limit) return rebuilt;
} else if (estimateCoreMessages(prepared.processedMessages) + imageTokens < limit) {
log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`);
return prepared;
}
// The payload still overflows the window: fail fast with a diagnostic
Expand All @@ -2617,7 +2628,7 @@ async function preflightCompressIfNeeded(
return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false };
}
const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502;
return failFast(status, f?.detail ?? "unknown reason", status === 503);
return failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", status === 503);
}

async function forward(
Expand Down
95 changes: 87 additions & 8 deletions tests/preflight-fail-fast.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ process.env.NODE_ENV = "test";
// exponential backoff — these tests are about the post-retry behavior.
process.env.BILI_REPLAY_RETRY_MAX = "1";

import { defaultConfig } from "acp-kernel";
import { defaultConfig, type Config } from "acp-kernel";
import { startServer, type ProxyOptions } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";
Expand Down Expand Up @@ -64,7 +64,30 @@ function makeUpstream429(calls?: Call[]): http.Server {
});
}

function startProxy(upstreamPort: number, models: Record<string, { context: number }>): Promise<http.Server> {
// Upstream that succeeds on BOTH the forwarded (stream) request and the
// preflight summarization (non-stream) call — needed to exercise the #330
// relaxed-zone fold path end to end.
function makeUpstreamOk(calls?: Call[]): http.Server {
return http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const raw = Buffer.concat(chunks).toString("utf8");
let parsed: { stream?: boolean } = {};
try { parsed = JSON.parse(raw); } catch { /* keep {} */ }
calls?.push({ stream: !!parsed.stream, body: raw });
if (parsed.stream) {
res.writeHead(200, { "content-type": "text/event-stream" });
res.end(okSse(1000));
} else {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ content: [{ type: "text", text: "SUMMARY: the large recent user message was a deterministic load-growth payload; its raw content is no longer needed." }] }));
}
});
});
}

function startProxy(upstreamPort: number, models: Record<string, { context: number }>, kernelOverrides?: Partial<Config>): Promise<http.Server> {
_setStoreForTest(new SessionStore({ enabled: false }));
setRegistryForTest({});
return startServer({
Expand All @@ -73,7 +96,7 @@ function startProxy(upstreamPort: number, models: Record<string, { context: numb
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models } },
modelContextLimit: 400_000,
kernelConfig: defaultConfig(400_000),
kernelConfig: defaultConfig(400_000, kernelOverrides),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
Expand Down Expand Up @@ -197,25 +220,36 @@ test("e2e #301: over-window payload with nothing compressible → structured 502
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } });
// #330: a lone large user message is now foldable (the soft recent zone is
// relaxed under overflow), so the truly-incompressible case is a large
// HARD-protected tool result: its paired tool_use (name "bash") is in
// protectedTools, so the result is excluded from every compressible range
// — even after the soft zone is relaxed.
const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }, { protectedTools: ["bash"] });
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
// A single ~12k-token message against a 10k window: over-window, but
// the kernel never folds the lone (first) user message → no
// compressible ranges → preflight cannot proceed at all.
const filler = "FILLER_".repeat(7000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-exhausted-sess" },
body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: filler }] }),
body: JSON.stringify({
model: "claude-small",
max_tokens: 1024,
stream: true,
messages: [
{ role: "assistant", content: [{ type: "tool_use", id: "call_1", name: "bash", input: { command: "echo hi" } }] },
{ role: "user", content: [{ type: "tool_result", tool_use_id: "call_1", content: filler }] },
],
}),
});
assert.equal(r.status, 502, "fail-fast 502 when nothing is compressible");
const json = JSON.parse(await r.text()) as { error?: { code?: string; message?: string; retryable?: boolean } };
assert.equal(json.error?.code, "preflight_compress_failed");
assert.equal(json.error?.retryable, false);
assert.ok(json.error?.message?.includes("NOT forwarded"), `message states the payload was withheld (got: ${json.error?.message})`);
assert.ok(json.error?.message?.includes("Raise the model context window"), `actionable error names the operator remedy (got: ${json.error?.message})`);
assert.equal(calls.filter((c) => !c.stream).length, 0, "no summarization call was spent on an incompressible payload");
assert.equal(calls.filter((c) => c.stream).length, 0, "the over-window payload was NOT forwarded upstream");
} finally {
Expand All @@ -225,3 +259,48 @@ test("e2e #301: over-window payload with nothing compressible → structured 502
await once(upstream, "close");
}
});

test("e2e #330: over-window payload whose only foldable content is in the protected recent zone → soft zone relaxed, folded, forwarded", async () => {
const calls: Call[] = [];
const upstream = makeUpstreamOk(calls);
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } });
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
// Three messages against a 10k window: a small opener plus two large
// recent messages (~8k tokens each, ~16k total). All three sit inside
// the soft-protected recent zone (preserveRecentMessages=5 covers all
// three), so the normal pass finds nothing foldable. Before #330 this
// 502'd forever; now preflight relaxes the soft zone, folds the oldest
// large message, and forwards the now-fitting payload.
const big1 = "A".repeat(32000);
const big2 = "B".repeat(32000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-relax-sess" },
body: JSON.stringify({
model: "claude-small",
max_tokens: 1024,
stream: true,
messages: [
{ role: "user", content: "start the task" },
{ role: "assistant", content: big1 },
{ role: "user", content: big2 },
],
}),
});
assert.equal(r.status, 200, "the over-window payload is folded and forwarded, not 502'd");
assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made the summarization call to fold the protected message");
assert.equal(calls.filter((c) => c.stream).length, 1, "the folded payload was forwarded upstream");
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});
Loading