Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions src/image-tokens.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// #488: images are invisible to the kernel's token model (wire codecs move them
// out of CoreMessage.text into sidecars), yet they ARE forwarded verbatim — so
// every fit decision built on the text-only estimate undercounts image-bearing
// payloads up to 15×. This module estimates the image share of a RAW request
// body so callers can add it to the text estimate. Known base64 payloads are
// charged ceil(base64Length/4) — the same chars/4 rule defaultCountTokens uses,
// matching how byte-counting relays actually bill (#488: 7 screenshots ≈ 1.4M
// tokens); remote URLs we cannot size get a flat conservative cost. Pixel-tile
// upstreams (official Anthropic/OpenAI) will see an overestimate — set
// BILI_IMAGE_TOKEN_CAP to clamp the per-image cost.

export const REMOTE_IMAGE_TOKENS = 4096;

function imageTokenCap(): number {
const v = Number(process.env.BILI_IMAGE_TOKEN_CAP ?? "");
return Number.isInteger(v) && v > 0 ? v : 0;
}

function applyCap(cost: number): number {
const cap = imageTokenCap();
return cap > 0 ? Math.min(cost, cap) : cost;
}

function costForUrl(url: string): number {
if (url.startsWith("data:")) {
const i = url.indexOf("base64,");
if (i >= 0) return applyCap(Math.ceil((url.length - i - "base64,".length) / 4));
}
return applyCap(REMOTE_IMAGE_TOKENS);
}

function isObj(v: unknown): v is Record<string, unknown> {
return typeof v === "object" && v !== null;
}

function urlOf(v: unknown): string | undefined {
if (typeof v === "string") return v;
if (isObj(v) && typeof v.url === "string") return v.url;
return undefined;
}

export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "responses", body: unknown): number {
if (!isObj(body)) return 0;
let total = 0;
if (protocol === "responses") {
const input = body.input;
if (!Array.isArray(input)) return 0;
for (const item of input) {
if (!isObj(item) || !Array.isArray(item.content)) continue;
for (const part of item.content) {
if (!isObj(part) || part.type !== "input_image") continue;
const url = urlOf(part.image_url);
if (url) total += costForUrl(url);
}
}
return total;
}
const messages = body.messages;
if (!Array.isArray(messages)) return 0;
for (const m of messages) {
if (!isObj(m) || !Array.isArray(m.content)) continue;
for (const part of m.content) {
if (!isObj(part)) continue;
if (protocol === "openai") {
if (part.type !== "image_url") continue;
const url = urlOf(part.image_url);
if (url) total += costForUrl(url);
} else {
if (part.type !== "image") continue;
const src = part.source;
if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += applyCap(Math.ceil(src.data.length / 4));
else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url);
}
}
}
return total;
}

// Cheap gate: most bodies carry no images — skip the JSON parse entirely then.
// prepared.body is bili's own compact JSON.stringify, but client raw buffers
// may carry spaces, so probe both forms.
export function imageTokensInRawBody(protocol: "anthropic" | "openai" | "responses", raw: string | Buffer): number {
const s = typeof raw === "string" ? raw : raw.toString("utf8");
const probe =
protocol === "responses" ? s.includes("input_image")
: protocol === "openai" ? s.includes("image_url")
: s.includes('"type":"image"') || s.includes('"type": "image"');
if (!probe) return 0;
try {
return imageTokensInParsedBody(protocol, JSON.parse(s));
} catch {
return 0;
}
}
11 changes: 7 additions & 4 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ export interface PreflightDeps {
proxyUrl?: string;
signal?: AbortSignal;
log: (level: string, msg: string) => void;
/** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */
imageFloor?: number;
}

export type PreflightFailureKind = "upstream" | "exhausted" | "aborted";
Expand Down Expand Up @@ -145,7 +147,8 @@ function summaryPayload(protocol: PreflightProtocol, model: string, system: stri
if (protocol === "openai") {
return { model, max_tokens: MAX_SUMMARY_OUTPUT_TOKENS, messages: [{ role: "system", content: system }, { role: "user", content }], stream: false };
}
return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false };
// #488: codex relays reject Responses calls without store:false ("Store must be set to false").
return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false, store: false };
}

function extractSummaryText(protocol: PreflightProtocol, json: Record<string, unknown>): string {
Expand Down Expand Up @@ -223,7 +226,7 @@ const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client

export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise<PreflightResult> {
const limit = deps.config.modelContextLimit;
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) };
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
if (limit <= 0) return result;
const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
// applyCompression rejects ranges below config.compress.minCompressRange
Expand Down Expand Up @@ -254,10 +257,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// Floor on the session's measured input baseline: the upstream's
// input_tokens also covers the system prompt + tool definitions, which
// are not in turn.messages, so the direct estimate can undershoot.
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages));
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
// The caller's forward/fail-fast gate uses the payload's own estimate
// (the floor can be stale — see PreflightResult.payloadEstimate).
result.payloadEstimate = estimateCoreMessages(turn.messages);
result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
if (startTokens < 0) startTokens = currentTokens;
if (currentTokens < limit) break;
const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
Expand Down
33 changes: 24 additions & 9 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import { COMPRESS_TOOL, ACP_TOOLS_ANTHROPIC, ACP_TOOLS_OPENAI, ACP_TOOLS_RESPONS
import { rewriteJsonResponse, type RewriteCtx } from "./stream.js";
import { applyRanges } from "./stream.js";
import { preflightCompress, estimateCoreMessages } from "./preflight.js";
import { imageTokensInRawBody, imageTokensInParsedBody } from "./image-tokens.js";
import { renderUI, handleConfigGet, handleConfigPut } from "./web/index.js";
import { reapOrphanBlocks } from "./orphan-gc.js";
import { getStore } from "./persist.js";
Expand Down Expand Up @@ -1559,13 +1560,15 @@ export function emergencyNudge(nudge: NudgeDecision | null | undefined, escalati
function clampOutgoingOutput(
rebuilt: Record<string, unknown>,
field: "max_tokens" | "max_completion_tokens" | "max_output_tokens",
ctx: { systemText: string; tools: unknown; processedMessages: CoreMessage[]; lastInputTokens: number; nativeWindow: number },
ctx: { systemText: string; tools: unknown; processedMessages: CoreMessage[]; lastInputTokens: number; nativeWindow: number; imageTokens: number },
sessionId: string,
log: (level: string, msg: string) => void,
): void {
const raw = rebuilt[field];
if (typeof raw !== "number") return;
const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens);
// #488: images ride along in the rebuilt body but are invisible to the text model —
// without them the cap is too generous and input+output can still overflow.
const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens) + ctx.imageTokens;
const capped = clampOutputBudget(raw, inputEstimate, ctx.nativeWindow);
if (capped !== undefined) {
rebuilt[field] = capped;
Expand Down Expand Up @@ -1675,7 +1678,7 @@ function prepareOpenai(
}

const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined };
clampOutgoingOutput(rebuilt as Record<string, unknown>, typeof (parsed as Record<string, unknown>).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log);
clampOutgoingOutput(rebuilt as Record<string, unknown>, typeof (parsed as Record<string, unknown>).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("openai", rebuilt) }, sessionId, log);
// prompt_cache_retention is an OpenAI-host-only cache directive; the dsh
// launcher forces PI_CACHE_RETENTION=long (for the session-id
// prompt_cache_key) which makes the client also emit it. Third-party
Expand Down Expand Up @@ -1884,7 +1887,7 @@ function prepareResponses(

const rebuilt: ResponsesRequestBody = { ...parsed, input: rebuiltInput, tools: toolsOut };
if (!isCompactionTrigger) {
clampOutgoingOutput(rebuilt as Record<string, unknown>, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log);
clampOutgoingOutput(rebuilt as Record<string, unknown>, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("responses", rebuilt) }, sessionId, log);
}
// Route with the upstream THIS request goes to — session.meta.upstreamOrigin
// is first-wins and would keep injecting pck toward a relay we switched
Expand Down Expand Up @@ -2292,7 +2295,10 @@ async function preflightCompressIfNeeded(
// A fresh session (id rotated, e.g. after a model switch) has
// lastInputTokens = 0 while still carrying a full raw history; size the
// trigger on the real post-fold payload too.
const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
// #488: images are forwarded verbatim but invisible to the kernel's text model —
// add their cost to every size decision here (trigger, fit gates, self-heal).
const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + imageTokens;
const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
if (limit <= 0 || !model || tokenCount < limit) return prepared;
// #301: forwarding as-is is safe ONLY when the payload's own estimate
Expand All @@ -2301,10 +2307,14 @@ async function preflightCompressIfNeeded(
// double-counted usage report (#300) — and must not turn a fitting
// payload into a fail-fast false positive.
const failFast = (status: number, detail: string, retryable: boolean): PreflightFailFast => {
const imageNote = imageTokens >= limit
? ` Images alone account for ~${imageTokens} tokens (≥ window ${limit}); compression cannot remove them — shrink or remove the images, or raise the window.`
: "";
const message =
`context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) ` +
`and preflight compression could not bring it under: ${detail}. ` +
`The over-window payload was NOT forwarded.`;
`and preflight compression could not bring it under: ${detail}.` +
imageNote +
` The over-window payload was NOT forwarded.`;
log("error", `[${session.id}] preflight fail-fast ${status} (retryable=${retryable}): ${message}`);
return { failFast: true, status, message, retryable, respond: !res.writableEnded };
};
Expand Down Expand Up @@ -2337,6 +2347,7 @@ async function preflightCompressIfNeeded(
proxyUrl,
signal: clientAbort.signal,
log,
imageFloor: imageTokens,
},
prepared.originalMessages,
);
Expand Down Expand Up @@ -2560,9 +2571,13 @@ async function forward(
// model that produced the overflow (per-model scoping: a stale
// limit from another model must not cap this one).
let reqModel: string | undefined;
let rejectedImageTokens = 0;
try {
const rawBody = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
reqModel = (JSON.parse(rawBody) as { model?: string }).model;
const parsedBody = JSON.parse(rawBody) as Record<string, unknown>;
reqModel = typeof parsedBody.model === "string" ? parsedBody.model : undefined;
// #488: the rejected payload's size must include its images (they were forwarded verbatim).
rejectedImageTokens = imageTokensInParsedBody(prepared.protocol, parsedBody);
} catch {
reqModel = undefined; // non-JSON body — fall back to the legacy scalar
}
Expand All @@ -2583,7 +2598,7 @@ async function forward(
// so the next turn re-centers the limit and the pre-flight
// compresses below it. Only shrink, never grow: a previously
// learned (smaller) value is the tighter bound.
const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + rejectedImageTokens;
const prev = (reqModel ? learnedMap[reqModel] : undefined) ?? (s.metadata.learnedContextLimit as number | undefined);
if (payloadEstimate >= 1000 && (prev === undefined || payloadEstimate < prev)) {
if (reqModel) learnedMap[reqModel] = payloadEstimate;
Expand Down
Loading
Loading