Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 125 additions & 13 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,13 @@ export interface PreflightFailure {
detail: string;
}

// #726: a summarization call can return HTTP 200 yet carry no usable summary
// text (in-stream error event, truncated stream, empty completion). The
// unusable branch carries a diagnosis of what the body actually contained so
// it is logged and surfaced in the fail-fast message instead of the generic
// "summary too short".
type SummaryOutcome = { summary: string } | { unusable: string };

export interface PreflightResult {
compressedRanges: number;
savedTokens: number;
Expand Down Expand Up @@ -167,6 +174,14 @@ function rangeChars(messages: CoreMessage[], startIdx: number, endIdx: number):
return chars;
}

function spanUnitsOf(messages: CoreMessage[], startIdx: number, endIdx: number, countText: (text: string) => number): number {
let units = 0;
for (let i = startIdx; i <= endIdx && i < messages.length; i++) {
units += countText(messages[i].text ?? "");
}
return units;
}

function renderRange(messages: CoreMessage[], startIdx: number, endIdx: number): string {
const parts: string[] = [];
for (let i = startIdx; i <= endIdx && i < messages.length; i++) {
Expand Down Expand Up @@ -359,7 +374,78 @@ function extractSummaryText(protocol: PreflightProtocol, json: Record<string, un
.join("");
}

async function summarizeRange(deps: PreflightDeps, content: string, startRef: string, endRef: string): Promise<string | null> {
// #726: an HTTP-200 summarization body can still be a rejection — the upstream
// may end its SSE stream with an error event (`error`, `response.failed`) or an
// incomplete response, or answer with a bare JSON error object. Without this
// scan such bodies are silently discarded and the only trace is "summary too
// short (0 chars)" with no way to tell size-driven from systemic failures.
function extractStreamError(o: Record<string, unknown>): string | null {
const t = typeof o.type === "string" ? o.type : undefined;
if (t === "error") {
const e = o.error;
if (e && typeof e === "object") {
const eo = e as Record<string, unknown>;
return `the upstream reported an in-stream error: ${typeof eo.message === "string" ? eo.message : JSON.stringify(eo).slice(0, 200)}`;
}
if (typeof o.message === "string") return `the upstream reported an in-stream error: ${o.message}`;
return "the upstream reported an in-stream error";
}
if (t === "response.failed" || t === "response.error") {
const resp = o.response;
if (resp && typeof resp === "object") {
const e = (resp as Record<string, unknown>).error;
if (e && typeof e === "object") {
const eo = e as Record<string, unknown>;
const code = typeof eo.code === "string" ? ` (${eo.code})` : "";
return `the upstream stream ended with a failed response${code}: ${typeof eo.message === "string" ? eo.message : JSON.stringify(eo).slice(0, 200)}`;
}
}
return "the upstream stream ended with a failed response";
}
if (t === "response.incomplete") {
const resp = (o.response ?? {}) as Record<string, unknown>;
const status = typeof resp.status === "string" ? resp.status : "unknown";
const e = resp.error as Record<string, unknown> | undefined;
const msg = e && typeof e.message === "string" ? ` (${e.message})` : "";
return `the upstream stream ended incomplete (status=${status}${msg})`;
}
if (!t && o.error && typeof o.error === "object") {
const eo = o.error as Record<string, unknown>;
return `the upstream reported an error: ${typeof eo.message === "string" ? eo.message : JSON.stringify(eo).slice(0, 200)}`;
}
return null;
}

export function diagnoseEmptySummary(text: string, json?: unknown): string {
if (json && typeof json === "object") {
const err = extractStreamError(json as Record<string, unknown>);
if (err) return err;
}
let sseEvents = 0;
let firstPayload = "";
for (const line of text.split("\n")) {
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (!payload || payload === "[DONE]") continue;
if (!firstPayload) firstPayload = payload.slice(0, 200);
let obj: unknown;
try {
obj = JSON.parse(payload);
} catch {
continue;
}
if (!obj || typeof obj !== "object") continue;
sseEvents += 1;
const err = extractStreamError(obj as Record<string, unknown>);
if (err) return err;
}
if (sseEvents > 0) return `the upstream stream carried ${sseEvents} SSE event(s) but no summary text (first event: ${firstPayload})`;
const trimmed = text.trim();
if (!trimmed) return "the upstream returned an empty body";
return `the upstream returned a non-SSE body with no summary text (first 200 bytes: ${trimmed.slice(0, 200)})`;
}

async function summarizeRange(deps: PreflightDeps, content: string, startRef: string, endRef: string): Promise<SummaryOutcome> {
const system =
buildCompressSystemPrompt(deps.prompts) +
`\n\nTASK: The conversation segment below (messages ${startRef}–${endRef}) must be compressed because the session context exceeds the current model's window. Write a tier-1 compression summary of the segment following every rule above. Output ONLY the summary text — no preamble, no closing remarks, no tool calls.`;
Expand Down Expand Up @@ -396,7 +482,7 @@ async function summarizeRange(deps: PreflightDeps, content: string, startRef: st
}
}

async function requestSummary(deps: PreflightDeps, system: string, content: string, stream: boolean, includeMaxOutputTokens: boolean): Promise<string | null> {
async function requestSummary(deps: PreflightDeps, system: string, content: string, stream: boolean, includeMaxOutputTokens: boolean): Promise<SummaryOutcome> {
const { response, clearTimer } = await fetchWithRetry(
deps.url,
{
Expand Down Expand Up @@ -431,10 +517,11 @@ async function requestSummary(deps: PreflightDeps, system: string, content: stri
deps.log("warn", `[preflight] summary response was not JSON: ${text.slice(0, 200)}`);
}
if (summary.length < MIN_SUMMARY_CHARS) {
deps.log("warn", `[preflight] summary too short (${summary.length} chars); skipping range`);
return null;
const diagnosis = diagnoseEmptySummary(text, json);
deps.log("warn", `[preflight] summary too short (${summary.length} chars): ${diagnosis}`);
return { unusable: diagnosis };
}
return summary;
return { summary };
} finally {
clearTimer();
}
Expand Down Expand Up @@ -486,6 +573,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
let finalUpper = baselineKnown ? 0 : estimateCoreMessagesUpper(messages);
let startTokens = -1;
let failure: PreflightFailure | undefined;
let lastUnusableDetail: string | undefined;
let activeConfig = deps.config;
let relaxed = false;
const relaxedExhaustedDetail =
Expand Down Expand Up @@ -581,13 +669,23 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// minUnits only in the char regime: with the optimistic token budget a
// sub-minimum chunk is already rare, and keeping minUnits = 0 there
// preserves the historical packing exactly.
for (const [cs, ce] of splitChunks(messages, startIdx, endIdx, budget, baselineKnown ? 0 : minChars, countText)) {
// #726: spans form a worklist instead of a flat pass. A chunk whose
// summary comes back unusable is halved (oldest half first) and
// retried down to a floor before the whole range is given up: the
// prime suspect for an empty summary is a CHUNK_FRACTION-sized chunk
// exceeding the upstream's real input cap, and halving recovers
// exactly those cases. Bounded by the per-regime call budget below.
const spans: Array<[number, number]> = splitChunks(messages, startIdx, endIdx, budget, baselineKnown ? 0 : minChars, countText).slice().reverse();
while (spans.length > 0) {
if (currentTokens < limit) break;
if (deps.signal?.aborted) {
failure = ABORTED_FAILURE;
break;
}
if (budgetHit) break;
const span = spans.pop();
if (!span) break;
const [cs, ce] = span;
const maps = refMaps(messages, deps.session.state);
const startRef = maps.idxToRef.get(cs);
const endRef = maps.idxToRef.get(ce);
Expand All @@ -600,9 +698,9 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
break;
}
summaryCalls += 1;
let summary: string | null;
let outcome: SummaryOutcome;
try {
summary = await summarizeRange(deps, content, startRef, endRef);
outcome = await summarizeRange(deps, content, startRef, endRef);
} catch (err) {
if (err instanceof UpstreamHttpError) {
failure = {
Expand All @@ -622,11 +720,21 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
}
break;
}
if (!summary) {
deps.log("warn", `[preflight] range ${skipKey} produced no usable summary; skipping it`);
if ("unusable" in outcome) {
lastUnusableDetail = outcome.unusable;
const floorUnits = baselineKnown ? 2 * MIN_CHUNK_TOKENS : 2 * minChars;
if (ce > cs && spanUnitsOf(messages, cs, ce, countText) >= floorUnits) {
deps.log("warn", `[preflight] chunk ${startRef}:${endRef} produced no usable summary (${outcome.unusable}); retrying with smaller chunks`);
const mid = Math.floor((cs + ce) / 2);
spans.push([mid + 1, ce]);
spans.push([cs, mid]);
continue;
}
deps.log("warn", `[preflight] range ${skipKey} produced no usable summary even at minimum size (${outcome.unusable}); skipping it`);
skipSet.add(skipKey);
break;
}
const summary = outcome.summary;
const ctx: RewriteCtx = {
core: deps.core,
config: activeConfig,
Expand Down Expand Up @@ -659,14 +767,18 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
if (appliedThisRound === 0) break;
}
if (currentTokens >= limit && !failure) {
// #726: carry the most recent unusable-summary diagnosis into the
// fail-fast message — "no range could be compressed" with no reason is
// undiagnosable from the client side.
const unusableNote = lastUnusableDetail ? ` Last unusable summary: ${lastUnusableDetail.slice(0, 300)}.` : "";
if (budgetHit) {
failure = { kind: "exhausted", detail: `the preflight summarization budget (${MAX_SUMMARY_CALLS_PER_PREFLIGHT} calls per protection regime) was exhausted before the payload fit the window` };
failure = { kind: "exhausted", detail: `the preflight summarization budget (${MAX_SUMMARY_CALLS_PER_PREFLIGHT} calls per protection regime) was exhausted before the payload fit the window${unusableNote}` };
} else if (relaxed && result.compressedRanges > 0) {
failure = { kind: "exhausted", detail: relaxedExhaustedDetail };
} else if (result.compressedRanges === 0) {
failure = { kind: "exhausted", detail: `no range could be compressed across ${rangesTried} viable range${rangesTried === 1 ? "" : "s"} (each was below minCompressRange, had an unusable summary, or failed to apply)` };
failure = { kind: "exhausted", detail: `no range could be compressed across ${rangesTried} viable range${rangesTried === 1 ? "" : "s"} (each was below minCompressRange, had an unusable summary, or failed to apply)${unusableNote}` };
} else {
failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds` };
failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds${unusableNote}` };
}
}
if (result.compressedRanges > 0) deps.session.stats.lastInputTokens = currentTokens;
Expand Down
50 changes: 49 additions & 1 deletion src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2972,6 +2972,23 @@ function preflightHoldGraceMs(): number {
return Number.isFinite(v) && v >= 0 ? Math.floor(v) : PREFLIGHT_HOLD_GRACE_DEFAULT_MS;
}

// #726: a preflight that failed WITHOUT folding anything hit a deterministic
// rejection under the current session state — a client that retries verbatim
// (codex auto-retries every few seconds) re-runs the same doomed walk, burning
// upstream quota on identical large summarization calls. Such failures arm a
// per-session cooldown: matching requests fail fast with the cached diagnosis
// and ZERO upstream calls until it expires. Any preflight outcome without a
// failure clears the marker (the state changed — client shrank the
// conversation, or the upstream recovered). Aborted failures never arm it.
const PREFLIGHT_DEAD_END_COOLDOWN_DEFAULT_MS = 5 * 60_000;

function preflightDeadEndCooldownMs(): number {
const raw = process.env.BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS;
if (!raw) return PREFLIGHT_DEAD_END_COOLDOWN_DEFAULT_MS;
const v = Number(raw);
return Number.isFinite(v) && v >= 0 ? Math.floor(v) : PREFLIGHT_DEAD_END_COOLDOWN_DEFAULT_MS;
}

/** #568: commit the response early so a long preflight cannot lose the client
* to its header timeout. Streaming clients get an SSE stream with keep-alive
* comment lines (`: bili-preflight` — a spec-mandated no-op for every SSE
Expand Down Expand Up @@ -3109,6 +3126,21 @@ async function preflightCompressIfNeeded(
// the only foldable content, and its exhaustion detail carries the
// operator remedy wording.
}
// #726: dead-end cooldown — an identical over-window state already failed
// preflight without folding anything, so re-running the walk is doomed; fail
// fast with the cached diagnosis and spend ZERO upstream summarization calls.
// Sits AFTER every safe-forward path above (#496 image arbitration, #300
// stale-baseline fit): those return without running the walk, so the
// cooldown must not convert a fitting payload into a false fail-fast while
// a marker from a larger earlier request is still warm.
const deadEnd = session.metadata.preflightDeadEnd;
if (deadEnd && typeof deadEnd === "object") {
const de = deadEnd as Record<string, unknown>;
if (typeof de.key === "string" && de.key === `${model}\u0000${limit}` && typeof de.until === "number" && de.until > Date.now() && typeof de.message === "string") {
log("warn", `[${session.id}] preflight dead-end cooldown active (${Math.ceil((de.until - Date.now()) / 1000)}s left); failing fast without upstream calls (#726)`);
return { failFast: true, status: typeof de.status === "number" ? de.status : 502, message: de.message, retryable: de.retryable === true, respond: !res.writableEnded };
}
}
// #330: the payload overflows the window (or nothing is foldable in the
// normal pass but it doesn't fit). Let preflightCompress try to fold it —
// it relaxes the soft-protected recent zone when nothing is foldable
Expand Down Expand Up @@ -3155,6 +3187,9 @@ async function preflightCompressIfNeeded(
clearTimeout(holdTimer);
stopHold?.();
}
// #726: a preflight that did not end in failure clears any dead-end marker
// — the state changed (conversation shrank, upstream recovered).
if (!result.failure) delete session.metadata.preflightDeadEnd;
// #330: decide forward/fail on the payload actually forwarded, not
// result.payloadEstimate — the preflight's relaxed-zone processTurn trims
// that estimate more than the normal-config prepare does, which can turn a
Expand Down Expand Up @@ -3187,7 +3222,20 @@ async function preflightCompressIfNeeded(
return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false };
}
const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502;
return failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", status === 503);
const ff = failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", status === 503);
// #726: zero-progress failure under unchanged state is a deterministic
// dead-end — arm the cooldown so client auto-retries stop re-burning
// upstream quota on the identical doomed summarization walk. Aborted
// failures never reach here.
if (f && result.compressedRanges === 0) {
const cooldownMs = preflightDeadEndCooldownMs();
if (cooldownMs > 0) {
ff.message += ` Preflight will not call the upstream again for the next ${Math.max(1, Math.round(cooldownMs / 60_000))}m while the context is unchanged (identical failure); restarting the session recovers immediately.`;
session.metadata.preflightDeadEnd = { key: `${model}\u0000${limit}`, until: Date.now() + cooldownMs, status: ff.status, retryable: ff.retryable, message: ff.message };
markDirty(session);
}
}
return ff;
}

/** #604: arm the emergency shrink after an upstream failure that will never
Expand Down
Loading
Loading