Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/image-tokens.ts
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,19 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp
}
continue;
}
// #3410: unknown top-level block types (document/PDF in the
// wild) ride the rawAnthropicBlock sidecar verbatim since this
// change — size inline base64 by payload, url refs by URL, and
// anything else at the flat remote floor (billing must see the
// preserved bytes or the fold gate undercounts). Known blocks
// (tool_use / string-content tool_result) carry no media bytes
// and fall through untouched, same as before.
if (part.type !== "text" && part.type !== "thinking" && part.type !== "tool_use" && part.type !== "tool_result") {
const src = part.source;
if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += base64ImageCost(src.data, billing, configuredCap);
else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url, billing, configuredCap);
else total += applyCap(REMOTE_IMAGE_TOKENS, configuredCap);
}
}
}
}
Expand All @@ -360,7 +373,7 @@ function bodyHasImagesProbe(protocol: "anthropic" | "openai" | "responses" | "go
return protocol === "google" ? s.includes("inlineData") || s.includes("fileData")
: protocol === "responses" ? s.includes("input_image") || s.includes('"content":[') || s.includes('"content": [')
: protocol === "openai" ? s.includes("image_url") || s.includes('"type":"file"') || s.includes('"type": "file"') || s.includes('"content":[') || s.includes('"content": [')
: s.includes('"type":"image"') || s.includes('"type": "image"') || s.includes('"type":"redacted_thinking"') || s.includes('"type":"document"');
: s.includes('"type":"image"') || s.includes('"type": "image"') || s.includes('"type":"redacted_thinking"') || s.includes('"type":"document"') || s.includes('"type": "document"');
}

export function imageTokensInRawBody(protocol: "anthropic" | "openai" | "responses" | "google", raw: string | Buffer, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number {
Expand Down Expand Up @@ -459,6 +472,10 @@ export function countImagesInParsedBody(protocol: "anthropic" | "openai" | "resp
}
continue;
}
// #3410: lockstep with the cost walker above — every sidecar-
// riding unknown block that bills there must count here too;
// known non-media blocks stay uncounted, same as before.
if (part.type !== "text" && part.type !== "thinking" && part.type !== "tool_use" && part.type !== "tool_result") n += 1;
}
}
}
Expand Down
18 changes: 17 additions & 1 deletion src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ import { DSH_COMPACTION_SHAPE_MSGS, dshCompactionRefusal, isDshCompactionCall }
import { clampOutgoingOutput, countSystemAndToolsTokens, emergencyNudge, estimateInputTokens, estimateWireOverhead, projectThinkingMass } from "./server/budget.js";
import { pipeThrough } from "./server/stream-io.js";
import { artifactSeedHit, detectAcpArtifacts } from "./server/chain-artifacts.js";
import { droppedOpenaiParts } from "./wire-drop-warn.js";
import { droppedOpenaiParts, unknownAnthropicBlocks } from "./wire-drop-warn.js";
import { BP_MAX, ROLLOUT_FEATURES, rolloutDecisions, rolloutInstallId, warnRolloutConfigProblems } from "./rollout.js";
export { forward } from "./server/relay.js";

Expand Down Expand Up @@ -188,6 +188,22 @@ export function warnDroppedOpenaiParts(parsed: unknown, sessionId: string, log:
log("warn", `[${sessionId}] wire codec will drop ${report.count} non-user content part(s) with unrecognized type(s) [${report.types.join(", ")}] (first at message #${report.firstIndex}) — kernel 0.0.85+ preserves all user-message parts (DeepSeek Files API refs included, #1205/#1188), but parts riding system/assistant/tool messages still reduce to text-only`);
}

// #3410: unknown anthropic top-level block types (document/PDF in the wild)
// are preserved verbatim since kernel 0.0.131 — nothing is dropped anymore,
// so this is a NOTICE, not a drop warning: one per session per type-set,
// same bounded FIFO as warnedWireDropKeys above.
export function warnUnknownAnthropicBlocks(parsed: unknown, sessionId: string, log: (level: string, msg: string) => void): void {
const report = unknownAnthropicBlocks(parsed);
if (!report) return;
const key = `${sessionId}:anthropic:${report.types.join(",")}`;
if (warnedWireDropKeys.has(key)) return;
warnedWireDropKeys.add(key);
if (warnedWireDropKeys.size > WARNED_WIREDROP_KEY_CAP) {
warnedWireDropKeys.delete(warnedWireDropKeys.values().next().value as string);
}
log("warn", `[${sessionId}] wire codec preserves ${report.count} anthropic block(s) with unrecognized type(s) [${report.types.join(", ")}] verbatim (first at message #${report.firstIndex}, #3410) — they ride the sidecar untouched but are excluded from context folding because bili cannot archive their payload`);
}

// #1284: upstream targets already warned about a "not a model conversation →
// relaying verbatim" verdict — misrouted third-party endpoints can retry in
// tight loops. Bounded FIFO, same shape as warnedWireDropKeys.
Expand Down
3 changes: 2 additions & 1 deletion src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { rewriteInboundThreadContinue } from "./thread-tracking.js";
import { captureThreadsTurn, estimateThreadsTokens, isThreadsBody, threadsReforgeEnabled } from "./threads-lane.js";
import type { CompressReasoningConfig } from "../reasoning-drop.js";
import type { ProxyOptions } from "../config.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, warnUnknownAnthropicBlocks, type Prepared } from "../server.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
Expand Down Expand Up @@ -199,6 +199,7 @@ export async function prepareAnthropic(
try {
const { msgs, cacheControls } = anthropicToCore(parsed);
originalMessages = msgs;
warnUnknownAnthropicBlocks(parsed, sessionId, log);
// #3756 snapshot render: outbound prefix from the unified transcript
// (fold-time ids, immune to identity re-mints); live tail keeps the
// client's bytes. No-op when the lane is inactive/degraded; the
Expand Down
41 changes: 39 additions & 2 deletions src/wire-drop-warn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function isObj(v: unknown): v is Record<string, unknown> {
return typeof v === "object" && v !== null;
}

interface DroppedOpenaiPartsReport {
interface WireDropReport {
count: number;
types: string[];
firstIndex: number;
Expand All @@ -25,7 +25,7 @@ interface DroppedOpenaiPartsReport {
// (stringContent), so only text survives there.
const DEFAULT_PRESERVED = new Set(["text"]);

export function droppedOpenaiParts(body: unknown): DroppedOpenaiPartsReport | null {
export function droppedOpenaiParts(body: unknown): WireDropReport | null {
if (!isObj(body)) return null;
const messages = body.messages;
if (!Array.isArray(messages)) return null;
Expand All @@ -45,3 +45,40 @@ export function droppedOpenaiParts(body: unknown): DroppedOpenaiPartsReport | nu
}
return count > 0 ? { count, types: [...types].sort(), firstIndex } : null;
}

// #3410: the anthropic codec used to drop every top-level block type outside
// the known six — in the wild today that means document/PDF blocks vanished
// on each rebuild with no trace. The kernel now rides unknown blocks on the
// rawAnthropicBlock sidecar and re-emits them verbatim, so nothing is dropped
// anymore; this detector scans the RAW body for those types so the proxy can
// log a one-time notice (the user may not know their client sent a PDF, and
// such blocks are excluded from folding since bili cannot archive them).
const ANTHROPIC_KNOWN_BLOCKS = new Set([
"text",
"tool_use",
"tool_result",
"thinking",
"image",
"redacted_thinking",
]);

export function unknownAnthropicBlocks(body: unknown): WireDropReport | null {
if (!isObj(body)) return null;
const messages = body.messages;
if (!Array.isArray(messages)) return null;
let count = 0;
const types = new Set<string>();
let firstIndex = -1;
for (let i = 0; i < messages.length; i++) {
const m = messages[i];
if (!isObj(m) || !Array.isArray(m.content)) continue;
for (const b of m.content) {
if (!isObj(b) || typeof b.type !== "string") continue;
if (ANTHROPIC_KNOWN_BLOCKS.has(b.type)) continue;
count++;
types.add(b.type);
if (firstIndex < 0) firstIndex = i;
}
}
return count > 0 ? { count, types: [...types].sort(), firstIndex } : null;
}
129 changes: 129 additions & 0 deletions tests/issue3410-unknown-anthropic-block.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import assert from "node:assert/strict";
import test from "node:test";

import { REMOTE_IMAGE_TOKENS, countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody } from "../src/image-tokens.ts";
import { unknownAnthropicBlocks } from "../src/wire-drop-warn.ts";
import { _resetWireDropWarningsForTest, warnUnknownAnthropicBlocks } from "../src/server.ts";

// #3410: unknown anthropic top-level block types (document/PDF in the wild)
// must be preserved verbatim by the kernel sidecar, excluded from folding
// (kernel protected predicates), noticed once per session per type-set, and
// priced like the openai file precedent so the fold gate cannot undercount.

const PDF_B64 = "JVBERi0xLjcKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcmM8PCRyb290IDEgMCBSDT4+CnN0YXJ0eHJlZgozMAolJUVPRg==";

const docBase64 = () => ({ type: "document", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } });
const docUrl = () => ({ type: "document", source: { type: "url", url: "https://files.example.com/report.pdf" } });
const docBare = () => ({ type: "document", source: { type: "text", text: "ref-only" } });
const imgUrl = () => ({ type: "image", source: { type: "url", url: "https://img.example.com/a.png" } });
const imgNoSource = () => ({ type: "image" });
const txt = () => ({ type: "text", text: "hello" });

function body(...content: unknown[]): Record<string, unknown> {
return { model: "claude-test", max_tokens: 100, messages: [{ role: "user", content }] };
}

test("unknownAnthropicBlocks detects unknown top-level block types across all roles", () => {
assert.equal(unknownAnthropicBlocks(null), null);
assert.equal(unknownAnthropicBlocks({}), null);
assert.equal(unknownAnthropicBlocks({ messages: "nope" }), null);
assert.equal(unknownAnthropicBlocks(body(txt(), imgUrl())), null);
assert.equal(unknownAnthropicBlocks(body({ role: "user", content: "plain string" } as never)), null);

const r = unknownAnthropicBlocks({
model: "claude-test",
messages: [
{ role: "assistant", content: [txt(), docBase64()] },
{ role: "user", content: [{ type: "custom_thing", v: 1 }] },
{ role: "user", content: [docUrl()] },
],
});
assert.deepEqual(r, { count: 3, types: ["custom_thing", "document"], firstIndex: 0 });
});

test("warnUnknownAnthropicBlocks logs once per session per distinct type-set", () => {
_resetWireDropWarningsForTest();
const lines: string[] = [];
const lg = (_level: string, msg: string) => lines.push(msg);

const d = body(docBase64());
warnUnknownAnthropicBlocks(d, "ses_a", lg);
warnUnknownAnthropicBlocks(d, "ses_a", lg); // same session, same type-set → deduped
assert.equal(lines.length, 1);
assert.match(lines[0], /\[ses_a\]/);
assert.match(lines[0], /document/);
assert.match(lines[0], /#3410/);

warnUnknownAnthropicBlocks(body({ type: "custom_thing", v: 1 }), "ses_a", lg); // new type-set → warns again
assert.equal(lines.length, 2);
warnUnknownAnthropicBlocks(d, "ses_b", lg); // new session → warns again
assert.equal(lines.length, 3);
warnUnknownAnthropicBlocks(body(txt(), imgUrl()), "ses_a", lg); // known types only → silent
assert.equal(lines.length, 3);
warnUnknownAnthropicBlocks(null, "ses_a", lg); // unparseable → silent
assert.equal(lines.length, 3);

_resetWireDropWarningsForTest();
warnUnknownAnthropicBlocks(d, "ses_a", lg); // after reset → warns again
assert.equal(lines.length, 4);
});

test("billing: anthropic document blocks are priced like the openai file precedent", () => {
const b64CostBytes = Math.ceil(PDF_B64.length / 4);

// base64 document: sized from its payload, identical to an image carrying the same bytes
const docB64Cost = imageTokensInParsedBody("anthropic", body(docBase64()));
assert.equal(docB64Cost, b64CostBytes);
assert.equal(imageTokensInParsedBody("anthropic", body({ type: "image", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } })), b64CostBytes);

// pixels mode: a PDF is not decodable as an image, so both fall back identically
const docPixels = imageTokensInParsedBody("anthropic", body(docBase64()), "pixels");
assert.equal(docPixels, imageTokensInParsedBody("anthropic", body({ type: "image", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } }), "pixels"));
assert.ok(docPixels > 0);

// url document: flat remote price
assert.equal(imageTokensInParsedBody("anthropic", body(docUrl())), REMOTE_IMAGE_TOKENS);
// bare/malformed-source document: flat remote price (never zero)
assert.equal(imageTokensInParsedBody("anthropic", body(docBare())), REMOTE_IMAGE_TOKENS);

// mixed content sums
assert.equal(imageTokensInParsedBody("anthropic", body(txt(), docUrl(), imgUrl())), 2 * REMOTE_IMAGE_TOKENS);

// regression anchors — image pricing is byte-for-byte what it was before #3410
assert.equal(imageTokensInParsedBody("anthropic", body(imgUrl())), REMOTE_IMAGE_TOKENS);
assert.equal(imageTokensInParsedBody("anthropic", body(imgNoSource())), 0);
assert.equal(imageTokensInParsedBody("anthropic", body(txt())), 0);
});

test("billing: count walker stays in lockstep with the cost walker for every document shape", () => {
const shapes: Array<[string, unknown]> = [
["base64", docBase64()],
["url", docUrl()],
["bare", docBare()],
["url image (anchor)", imgUrl()],
["sourceless image (anchor)", imgNoSource()],
["text (anchor)", txt()],
];
for (const [name, part] of shapes) {
const cost = imageTokensInParsedBody("anthropic", body(part));
const count = countImagesInParsedBody("anthropic", body(part));
assert.equal(count > 0, cost > 0, `${name}: cost/count sign mismatch (cost=${cost}, count=${count})`);
assert.ok(count <= 1, `${name}: expected at most one media part, got ${count}`);
}
assert.equal(countImagesInParsedBody("anthropic", body(docBase64(), docUrl(), docBare())), 3);
});

test("billing: the raw-body probe admits document-only bodies (compact and spaced JSON)", () => {
const compact = JSON.stringify(body(docBase64()));
assert.ok(compact.includes('"type":"document"'));
assert.equal(imageTokensInRawBody("anthropic", compact), Math.ceil(PDF_B64.length / 4));
assert.equal(countImagesInRawBody("anthropic", compact), 1);

const spaced = compact.replace(/"type":"/g, '"type": "');
assert.ok(spaced.includes('"type": "document"'));
assert.equal(imageTokensInRawBody("anthropic", spaced), Math.ceil(PDF_B64.length / 4));

// clean bodies stay free — the probe must not over-admit
assert.equal(imageTokensInRawBody("anthropic", JSON.stringify(body(txt()))), 0);
assert.equal(countImagesInRawBody("anthropic", JSON.stringify(body(txt()))), 0);
});
Loading