Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions src/egress-shed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,66 @@ export async function shedWireImages(
};
}

/** #4124: shed images from a serialized request until at most maxImages remain.
* Count-space twin of shedWireImages — RUNG A ONLY, because archiving is the only
* action that reduces COUNT (rung B downscaling leaves the image count untouched).
* Same placeholder/archive machinery, so every original stays reachable via
* decompress({imageRef}). Two deliberate differences from shedWireImages:
* - the work list is NOT sliced at MAX_SHED_IMAGES — that bound caps jimp CPU,
* while archiving is a cheap string swap and a 165→29 convergence needs ~136;
* - the protected tail YIELDS to the cap: an upstream count wall is HARD (sending
* over it fails the whole request), unlike the WS byte budget which is soft —
* once non-tail images are exhausted, tail images are archived too (oldest-first).
* Exempt read_original_image occurrences never enter the walk (collectOccurrences
* skips them); if exempt images alone exceed the cap the body cannot be brought
* under it and the rejection stays visible. Pure on the string; byte-identical
* passthrough when the count is already within maxImages or nothing was sheddable. */
export async function shedWireImagesToCount(
body: string,
protocol: WireProtocol,
maxImages: number,
opts: ShedOptions = {},
): Promise<{ body: string; stats: ShedStats }> {
const originalBytes = byteLen(body);
const noop: ShedStats = { originalBytes, finalBytes: originalBytes, archived: 0, downscaled: 0, changed: false };
if (!Number.isInteger(maxImages) || maxImages < 0) return { body, stats: noop };
let parsed: unknown;
try {
parsed = JSON.parse(body);
} catch {
return { body, stats: noop };
}
if (!isObj(parsed)) return { body, stats: noop };
const occurrences = collectOccurrences(protocol, parsed);
if (occurrences.length <= maxImages) return { body, stats: noop };

const totalEntries = protocol === "google" ? (Array.isArray(parsed.contents) ? parsed.contents.length : 0)
: protocol === "responses" ? (Array.isArray(parsed.input) ? parsed.input.length : 0)
: Array.isArray(parsed.messages) ? parsed.messages.length : 0;
const protectFrom = Math.max(0, totalEntries - PROTECTED_TAIL_ENTRIES);
for (const o of occurrences) o.protectedTail = o.entryIndex >= protectFrom;

let remaining = occurrences.length;
let archived = 0;
for (const allowTail of [false, true]) {
if (remaining <= maxImages) break;
for (const o of occurrences) {
if (remaining <= maxImages) break;
if (o.archived) continue;
if (!allowTail && o.protectedTail) continue;
const note = await shedNote(o.mediaType, o.b64, opts.resolveRef);
o.applyArchive(note);
o.archived = true;
remaining--;
archived++;
}
}

if (archived === 0) return { body, stats: noop };
const next = JSON.stringify(parsed);
return { body: next, stats: { originalBytes, finalBytes: byteLen(next), archived, downscaled: 0, changed: true } };
}

interface ShedContext {
sessionId?: string;
session?: Session;
Expand Down
108 changes: 105 additions & 3 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import { buildAbsorbSystemPrompt, buildCompressHybridSystemPrompt, buildCompress
import { findRoute, resolveDeclaredProtocol, type ProxyOptions } from "../config.js";
import { makeContinuationRefetch } from "../degenerate-retry.js";
import { dumpRejectedBody } from "../error-dump.js";
import { extractWsOversizeError, formatMiB, shedEgressIfOverBudget, shedWireImages, type ShedStats } from "../egress-shed.js";
import { extractWsOversizeError, formatMiB, shedEgressIfOverBudget, shedWireImages, shedWireImagesToCount, type ShedStats } from "../egress-shed.js";
import { conversationPromptSections, externalSummaryEnabled } from "../external-summary-surface.js";
import { fakeBufCap, injectFakeCompletionHint, isFakeCompletion, maxFakeCompletionRetries } from "../fake-completion.js";
import { fetchWithTimeout, fetchWithTransportRetry } from "../fetch-util.js";
Expand Down Expand Up @@ -552,6 +552,24 @@ export async function forward(
if (prepared && !prepared.sidePassthrough && prepared.protocol && req.method !== "GET" && req.method !== "HEAD" && typeof wireBody === "string") {
const shed = await shedEgressIfOverBudget(wireBody, prepared.protocol, { sessionId: prepared.session.id, session: prepared.session, log, phase: "pre-send" });
wireBody = shed.body;
// #4124: proactive per-prompt image-count cap — once this route PROVED a
// count wall via an upstream rejection (learned into stats.imageCountCaps),
// converge the outbound copy under it BEFORE send so the request never 400s
// on count again. Only acts when the count is over; under-cap bodies pass
// through byte-identical (deterministic walk → prefix-cache stable).
const imgOrigin = normalizeUpstreamOrigin(upstreamUrl);
const imgCap = imgOrigin !== undefined ? prepared.session.stats.imageCountCaps?.[imgOrigin] : undefined;
if (typeof imgCap === "number" && imgCap > 0) {
const sess = prepared.session;
const countShed = await shedWireImagesToCount(wireBody, prepared.protocol, imgCap - 1, {
resolveRef: (mt, b64) => refForItemBytes(sess, mt, b64),
});
if (countShed.stats.changed) {
sess.metadata.imageCountShed = { at: Date.now(), phase: "pre-send", cap: imgCap, archived: countShed.stats.archived };
log("info", `[${sess.id}] [image-count-shed] pre-send: archived ${countShed.stats.archived} oldest images to stay under the learned per-prompt cap ${imgCap} on this route (#4124) — originals stay in session storage`);
wireBody = countShed.body;
}
}
}
// #1884: sign the FINAL wire body right before the send — everything
// upstream of this point (prepare* injection, compat, steering) already
Expand Down Expand Up @@ -1002,7 +1020,17 @@ export async function forward(
} catch {
strippedPckBody = null;
}
if (strippedPckBody !== null) {
// #4124: a 400 whose body names a CONTENT wall (image-count or
// token/context overflow) exonerates prompt_cache_key by inspection —
// the key-less re-send can only fail identically, and in the field it
// hit a secondary byte-wall 413 instead (#4124 turn 44). Skip the
// probe; the overflow paths below own this error.
const pckInspection = strippedPckBody !== null ? inspectContextOverflow(400, pckErrText) : null;
const contentWall = pckInspection !== null && (pckInspection.isOverflow || pckInspection.imageCountCap !== undefined);
if (contentWall) {
log("debug", `[${prepared.session.id}] [pck] 400 names a content wall (${pckInspection!.imageCountCap !== undefined ? `image count > ${pckInspection!.imageCountCap}` : "context overflow"}) — prompt_cache_key exonerated by inspection, A/B probe skipped (#4124)`);
}
if (strippedPckBody !== null && !contentWall) {
let retried: Awaited<ReturnType<typeof fetchWithTimeout>> | null = null;
try {
applyResign(headers, strippedPckBody);
Expand Down Expand Up @@ -1219,7 +1247,67 @@ export async function forward(
stopIdleTimer: upstreamResult.stopIdleTimer,
};
const overflowInfo = inspectContextOverflow(upstreamResult.response.status, overflowText);
if (overflowInfo.isOverflow) {
if (overflowInfo.imageCountCap !== undefined && prepared!.protocol && typeof wireBody === "string") {
// #4124: image-COUNT wall — a different dimension than the token
// overflow below; armOverflowShrink must NOT run (its no-window
// branch would arm a bogus token-space shrink at the payload
// estimate). Learn the cap per-origin, converge the outbound copy
// to cap−1 with the count-shed machinery, and re-send ONCE within
// this same request. An unchanged body (nothing archivable), a
// Buffer body (raw passthrough bytes), or a missing protocol falls
// through to the verbatim passthrough below, which still learns
// the cap there.
const s = prepared!.session;
const imgOrigin = normalizeUpstreamOrigin(upstreamUrl);
if (imgOrigin !== undefined) {
const caps = s.stats.imageCountCaps ?? {};
caps[imgOrigin] = overflowInfo.imageCountCap;
s.stats.imageCountCaps = caps;
markDirty(s);
}
const target = Math.max(0, overflowInfo.imageCountCap - 1);
const countShed = await shedWireImagesToCount(wireBody, prepared!.protocol, target, {
resolveRef: (mt, b64) => refForItemBytes(s, mt, b64),
});
if (!countShed.stats.changed) {
log("warn", `[${s.id}] image-count overflow (upstream cap ${overflowInfo.imageCountCap}, request carried ${overflowInfo.imageCountSeen ?? "?"}) — nothing archivable in the outbound copy (images sit in exempt tool outputs?) — passing the rejection through (#4124)`);
} else {
try {
s.metadata.imageCountShed = { at: Date.now(), phase: "same-request-retry", cap: overflowInfo.imageCountCap, archived: countShed.stats.archived };
applyResign(headers, countShed.body);
const retried = await fetchWithTimeout(upstreamUrl, { ...init, body: v1Wrap(countShed.body) }, undefined, clientAbort.signal);
if (retried.response.ok) {
upstreamResult.clearTimer();
wireBody = countShed.body; // track the accepted re-send as the wire base
upstreamResult = retried;
log("info", `[${s.id}] image-count overflow (upstream cap ${overflowInfo.imageCountCap}${overflowInfo.imageCountSeen !== undefined ? `, request carried ${overflowInfo.imageCountSeen}` : ""}) — archived ${countShed.stats.archived} oldest images, re-sent within the same request, upstream accepted (#4124)`);
} else {
let retryErrText: string | null = null;
if (retried.response.body) {
try {
retryErrText = (await readStreamToBuffer(retried.response.body)).toString("utf8");
} catch {
retryErrText = null;
}
}
// Answer with the retry's own verdict, not the stale first 400.
upstreamResult.clearTimer();
upstreamResult = {
response: new Response(retryErrText ?? "", {
status: retried.response.status,
statusText: retried.response.statusText,
headers: new Headers(retried.response.headers),
}),
clearTimer: retried.clearTimer,
stopIdleTimer: retried.stopIdleTimer,
};
log("warn", `[${s.id}] image-count overflow — converged to ${target} images but the re-send was still rejected (HTTP ${retried.response.status}; upstream counted ${overflowInfo.imageCountSeen ?? "?"}) — passing the retry verdict through (#4124)`);
}
} catch {
// transport failure — the buffered original 400 answers below
}
}
} else if (overflowInfo.isOverflow) {
armOverflowShrink(overflowInfo);
// #3787 follow-up: a NUMBERLESS overflow states no window; the
// armed baseline (min of declared and the rejected payload's
Expand Down Expand Up @@ -1350,6 +1438,20 @@ export async function forward(
const info = inspectContextOverflow(upstream.status, errBody.toString("utf8"));
if (info.isOverflow) {
armOverflowShrink(info);
} else if (info.imageCountCap !== undefined) {
// #4124: an image-count wall reached this path without the
// same-request rescue above (no refold hook / response not
// writable / no protocol) — still LEARN the cap per-origin so
// the next request on this route converges pre-send. Never arms
// the token-space shrink (a count is not tokens).
const imgOrigin = normalizeUpstreamOrigin(upstreamUrl);
if (imgOrigin !== undefined) {
const caps = s.stats.imageCountCaps ?? {};
caps[imgOrigin] = info.imageCountCap;
s.stats.imageCountCaps = caps;
markDirty(s);
}
log("warn", `[${s.id}] upstream image-count wall (cap ${info.imageCountCap}, request carried ${info.imageCountSeen ?? "?"}) — learned for pre-send convergence on this route; no token-space arm (#4124)`);
}
// #762: learn strict-echo on the MAIN request path too. The loop-only
// learner (src/loop/core.ts) never sees client-originated 400s, so a
Expand Down
9 changes: 9 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,15 @@ export type Session = {
/** #3450: one-time mismatch warning already emitted for this session
* (upstream-stated window ≠ window in use). Provenance only. */
overflowArmMismatchWarned?: boolean;
/** #4124: per-origin learned per-prompt image-count caps, from upstream
* rejections ("Too many images in request: N > M"). Key =
* normalizeUpstreamOrigin(url) — the same origin stamping as the token
* overflow arm (#2313), because the cap belongs to the ROUTE, not the
* session. Separate from overflowArmTokens by construction: that channel
* rejects sub-1000 numbers and drives TOKEN-space shrink; an image cap
* is a count. Persisted with the session; no TTL — a stale (too low) cap
* can only over-shed, never under-shed. */
imageCountCaps?: Record<string, number>;
/** Tokens compressed THIS turn whose fold has not yet materialized in
* an upstream usage report (the post-compress re-request re-sends the
* UNFOLDED history for prefix-cache reasons, so its usage report
Expand Down
70 changes: 64 additions & 6 deletions src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,15 @@ export interface ContextOverflowInfo {
* provider bills OUTPUT against the same wall as input, so preflight
* must fold against window − reservation. */
requestedCompletion?: number;
/** #4124: per-prompt IMAGE-COUNT hard cap stated in the error body
* ("Too many images in request: 165 > 30"). A COUNT, not tokens — it must
* never ride the window/lastInputTokens channel (toTokenNumber rejects
* sub-1000 numbers, and that channel drives TOKEN-space shrink). Presence
* marks an image-count overflow; isOverflow stays token/context-only. */
imageCountCap?: number;
/** #4124: the request's own image count as the upstream counted it ("165")
* — cross-check/log evidence only; bili counts occurrences itself. */
imageCountSeen?: number;
/** Truncated error-body text, for logging. */
message: string;
}
Expand Down Expand Up @@ -332,6 +341,45 @@ function toTokenNumber(s: string): number | undefined {
return Number.isFinite(n) && n >= 1000 ? n : undefined;
}

// #4124: per-prompt IMAGE-COUNT walls — a different dimension from the token
// patterns above. vLLM (--limit-mm-per-prompt) and gateway equivalents reject
// by COUNT of images, so a payload far under its token window still 400s on
// every round: images meter into tokens, but no fold decision ever looks at
// count. The cap is small (tens), which is exactly why it cannot ride the
// token-window channel (toTokenNumber rejects <1000; that channel drives
// token-space shrink). Known shapes:
// vLLM: "Too many images in request: 165 > 30" — gateways re-serialize error
// bodies with HTML-style escapes, so the wire form of ">" is "\u003e"
// (#4124 field log); both spellings must match. $1 = the request's own
// count (cross-check only), $2 = the cap.
// generic: "At most N image(s) may be provided" — cap only.
const IMAGE_COUNT_OVERFLOW_PATTERNS: RegExp[] = [
/too many images in request:\s*(\d+)\s*(?:>|\\u003e)\s*(\d+)/i,
/at most\s+(\d+)\s+images?\s+may be provided/i,
];

/** #4124: extract the per-prompt image-count cap (+ the request's own count,
* when the body states both) from an error body. Returns an empty object
* unless a positive finite cap is present — a wrong guess would shed images
* the upstream actually accepts. */
function parseImageCountCap(text: string): { cap?: number; seen?: number } {
let m = text.match(IMAGE_COUNT_OVERFLOW_PATTERNS[0]);
if (m) {
const seen = Number(m[1]);
const cap = Number(m[2]);
if (Number.isFinite(cap) && cap > 0) {
return { cap, ...(seen > 0 ? { seen } : {}) };
}
return {};
}
m = text.match(IMAGE_COUNT_OVERFLOW_PATTERNS[1]);
if (m) {
const cap = Number(m[1]);
if (Number.isFinite(cap) && cap > 0) return { cap };
}
return {};
}

/** Best-effort extraction of the real context window from an overflow error
* body. Returns undefined when no confident window number is present — a wrong
* guess (e.g. the prompt size, not the limit) is worse than no guess. */
Expand Down Expand Up @@ -416,14 +464,24 @@ export function inspectContextOverflow(status: number, bodyText: string): Contex
if (!plausible) return { isOverflow: false, message };
if (!bodyText) return { isOverflow: false, message };
const isOverflow = CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(bodyText));
if (!isOverflow) return { isOverflow: false, message };
// #4124: the image-count wall is checked INDEPENDENTLY of the token markers
// — a "Too many images" body matches none of them (and must not arm any
// token-space machinery), yet every call site sees ONE inspection result.
const imageCount = parseImageCountCap(bodyText);
if (!isOverflow && imageCount.cap === undefined) return { isOverflow: false, message };
const counts = parseOverflowCounts(bodyText);
return {
isOverflow: true,
window: parseOverflowWindow(bodyText),
...(counts.total !== undefined ? { requestedTotal: counts.total } : {}),
...(counts.messages !== undefined ? { requestedMessages: counts.messages } : {}),
...(counts.completion !== undefined ? { requestedCompletion: counts.completion } : {}),
isOverflow,
...(imageCount.cap !== undefined ? { imageCountCap: imageCount.cap } : {}),
...(imageCount.seen !== undefined ? { imageCountSeen: imageCount.seen } : {}),
...(isOverflow
? {
window: parseOverflowWindow(bodyText),
...(counts.total !== undefined ? { requestedTotal: counts.total } : {}),
...(counts.messages !== undefined ? { requestedMessages: counts.messages } : {}),
...(counts.completion !== undefined ? { requestedCompletion: counts.completion } : {}),
}
: {}),
message,
};
}
Expand Down
Loading
Loading