diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 91d85e16b9..cd2a096970 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -342,6 +342,19 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp } continue; } + // #3410: unknown top-level block types (document/PDF in the + // wild) ride the rawAnthropicBlock sidecar verbatim since this + // change — size inline base64 by payload, url refs by URL, and + // anything else at the flat remote floor (billing must see the + // preserved bytes or the fold gate undercounts). Known blocks + // (tool_use / string-content tool_result) carry no media bytes + // and fall through untouched, same as before. + if (part.type !== "text" && part.type !== "thinking" && part.type !== "tool_use" && part.type !== "tool_result") { + const src = part.source; + if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += base64ImageCost(src.data, billing, configuredCap); + else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url, billing, configuredCap); + else total += applyCap(REMOTE_IMAGE_TOKENS, configuredCap); + } } } } @@ -360,7 +373,7 @@ function bodyHasImagesProbe(protocol: "anthropic" | "openai" | "responses" | "go return protocol === "google" ? s.includes("inlineData") || s.includes("fileData") : protocol === "responses" ? s.includes("input_image") || s.includes('"content":[') || s.includes('"content": [') : protocol === "openai" ? s.includes("image_url") || s.includes('"type":"file"') || s.includes('"type": "file"') || s.includes('"content":[') || s.includes('"content": [') - : s.includes('"type":"image"') || s.includes('"type": "image"') || s.includes('"type":"redacted_thinking"') || s.includes('"type":"document"'); + : s.includes('"type":"image"') || s.includes('"type": "image"') || s.includes('"type":"redacted_thinking"') || s.includes('"type":"document"') || s.includes('"type": "document"'); } export function imageTokensInRawBody(protocol: "anthropic" | "openai" | "responses" | "google", raw: string | Buffer, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number { @@ -459,6 +472,10 @@ export function countImagesInParsedBody(protocol: "anthropic" | "openai" | "resp } continue; } + // #3410: lockstep with the cost walker above — every sidecar- + // riding unknown block that bills there must count here too; + // known non-media blocks stay uncounted, same as before. + if (part.type !== "text" && part.type !== "thinking" && part.type !== "tool_use" && part.type !== "tool_result") n += 1; } } } diff --git a/src/server.ts b/src/server.ts index db05659ab1..680ce19c11 100644 --- a/src/server.ts +++ b/src/server.ts @@ -155,7 +155,7 @@ import { DSH_COMPACTION_SHAPE_MSGS, dshCompactionRefusal, isDshCompactionCall } import { clampOutgoingOutput, countSystemAndToolsTokens, emergencyNudge, estimateInputTokens, estimateWireOverhead, projectThinkingMass, readOutputBudget } from "./server/budget.js"; import { pipeThrough } from "./server/stream-io.js"; import { artifactSeedHit, detectAcpArtifacts } from "./server/chain-artifacts.js"; -import { droppedOpenaiParts } from "./wire-drop-warn.js"; +import { droppedOpenaiParts, unknownAnthropicBlocks } from "./wire-drop-warn.js"; import { BP_MAX, ROLLOUT_FEATURES, rolloutDecisions, rolloutInstallId, warnRolloutConfigProblems } from "./rollout.js"; export { forward } from "./server/relay.js"; @@ -188,6 +188,22 @@ export function warnDroppedOpenaiParts(parsed: unknown, sessionId: string, log: log("warn", `[${sessionId}] wire codec will drop ${report.count} non-user content part(s) with unrecognized type(s) [${report.types.join(", ")}] (first at message #${report.firstIndex}) — kernel 0.0.85+ preserves all user-message parts (DeepSeek Files API refs included, #1205/#1188) and mid-conversation system/developer messages ride verbatim (#4166), but parts riding assistant/tool messages and the leading system/developer run still reduce to text-only`); } +// #3410: unknown anthropic top-level block types (document/PDF in the wild) +// are preserved verbatim since kernel 0.0.131 — nothing is dropped anymore, +// so this is a NOTICE, not a drop warning: one per session per type-set, +// same bounded FIFO as warnedWireDropKeys above. +export function warnUnknownAnthropicBlocks(parsed: unknown, sessionId: string, log: (level: string, msg: string) => void): void { + const report = unknownAnthropicBlocks(parsed); + if (!report) return; + const key = `${sessionId}:anthropic:${report.types.join(",")}`; + if (warnedWireDropKeys.has(key)) return; + warnedWireDropKeys.add(key); + if (warnedWireDropKeys.size > WARNED_WIREDROP_KEY_CAP) { + warnedWireDropKeys.delete(warnedWireDropKeys.values().next().value as string); + } + log("warn", `[${sessionId}] wire codec preserves ${report.count} anthropic block(s) with unrecognized type(s) [${report.types.join(", ")}] verbatim (first at message #${report.firstIndex}, #3410) — they ride the sidecar untouched but are excluded from context folding because bili cannot archive their payload`); +} + // #1284: upstream targets already warned about a "not a model conversation → // relaying verbatim" verdict — misrouted third-party endpoints can retry in // tight loops. Bounded FIFO, same shape as warnedWireDropKeys. diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index c9c773d6e5..99a1df2297 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -6,7 +6,7 @@ import { rewriteInboundThreadContinue } from "./thread-tracking.js"; import { captureThreadsTurn, estimateThreadsTokens, isThreadsBody, threadsReforgeEnabled } from "./threads-lane.js"; import type { CompressReasoningConfig } from "../reasoning-drop.js"; import type { ProxyOptions } from "../config.js"; -import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js"; +import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, warnUnknownAnthropicBlocks, type Prepared } from "../server.js"; import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; @@ -200,6 +200,7 @@ export async function prepareAnthropic( try { const { msgs, cacheControls } = anthropicToCore(parsed); originalMessages = msgs; + warnUnknownAnthropicBlocks(parsed, sessionId, log); // #3756 snapshot render: outbound prefix from the unified transcript // (fold-time ids, immune to identity re-mints); live tail keeps the // client's bytes. No-op when the lane is inactive/degraded; the diff --git a/src/wire-drop-warn.ts b/src/wire-drop-warn.ts index 8f37e1375e..676e40ccc9 100644 --- a/src/wire-drop-warn.ts +++ b/src/wire-drop-warn.ts @@ -18,7 +18,7 @@ function isObj(v: unknown): v is Record { return typeof v === "object" && v !== null; } -interface DroppedOpenaiPartsReport { +interface WireDropReport { count: number; types: string[]; firstIndex: number; @@ -29,7 +29,7 @@ interface DroppedOpenaiPartsReport { // survives there (mid system/developer ride the rawOpenaiMessage sidecar, #4166). const DEFAULT_PRESERVED = new Set(["text"]); -export function droppedOpenaiParts(body: unknown): DroppedOpenaiPartsReport | null { +export function droppedOpenaiParts(body: unknown): WireDropReport | null { if (!isObj(body)) return null; const messages = body.messages; if (!Array.isArray(messages)) return null; @@ -54,3 +54,40 @@ export function droppedOpenaiParts(body: unknown): DroppedOpenaiPartsReport | nu } return count > 0 ? { count, types: [...types].sort(), firstIndex } : null; } + +// #3410: the anthropic codec used to drop every top-level block type outside +// the known six — in the wild today that means document/PDF blocks vanished +// on each rebuild with no trace. The kernel now rides unknown blocks on the +// rawAnthropicBlock sidecar and re-emits them verbatim, so nothing is dropped +// anymore; this detector scans the RAW body for those types so the proxy can +// log a one-time notice (the user may not know their client sent a PDF, and +// such blocks are excluded from folding since bili cannot archive them). +const ANTHROPIC_KNOWN_BLOCKS = new Set([ + "text", + "tool_use", + "tool_result", + "thinking", + "image", + "redacted_thinking", +]); + +export function unknownAnthropicBlocks(body: unknown): WireDropReport | null { + if (!isObj(body)) return null; + const messages = body.messages; + if (!Array.isArray(messages)) return null; + let count = 0; + const types = new Set(); + let firstIndex = -1; + for (let i = 0; i < messages.length; i++) { + const m = messages[i]; + if (!isObj(m) || !Array.isArray(m.content)) continue; + for (const b of m.content) { + if (!isObj(b) || typeof b.type !== "string") continue; + if (ANTHROPIC_KNOWN_BLOCKS.has(b.type)) continue; + count++; + types.add(b.type); + if (firstIndex < 0) firstIndex = i; + } + } + return count > 0 ? { count, types: [...types].sort(), firstIndex } : null; +} diff --git a/tests/issue3410-unknown-anthropic-block.test.ts b/tests/issue3410-unknown-anthropic-block.test.ts new file mode 100644 index 0000000000..5110d4009a --- /dev/null +++ b/tests/issue3410-unknown-anthropic-block.test.ts @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { REMOTE_IMAGE_TOKENS, countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody } from "../src/image-tokens.ts"; +import { unknownAnthropicBlocks } from "../src/wire-drop-warn.ts"; +import { _resetWireDropWarningsForTest, warnUnknownAnthropicBlocks } from "../src/server.ts"; + +// #3410: unknown anthropic top-level block types (document/PDF in the wild) +// must be preserved verbatim by the kernel sidecar, excluded from folding +// (kernel protected predicates), noticed once per session per type-set, and +// priced like the openai file precedent so the fold gate cannot undercount. + +const PDF_B64 = "JVBERi0xLjcKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcmM8PCRyb290IDEgMCBSDT4+CnN0YXJ0eHJlZgozMAolJUVPRg=="; + +const docBase64 = () => ({ type: "document", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } }); +const docUrl = () => ({ type: "document", source: { type: "url", url: "https://files.example.com/report.pdf" } }); +const docBare = () => ({ type: "document", source: { type: "text", text: "ref-only" } }); +const imgUrl = () => ({ type: "image", source: { type: "url", url: "https://img.example.com/a.png" } }); +const imgNoSource = () => ({ type: "image" }); +const txt = () => ({ type: "text", text: "hello" }); + +function body(...content: unknown[]): Record { + return { model: "claude-test", max_tokens: 100, messages: [{ role: "user", content }] }; +} + +test("unknownAnthropicBlocks detects unknown top-level block types across all roles", () => { + assert.equal(unknownAnthropicBlocks(null), null); + assert.equal(unknownAnthropicBlocks({}), null); + assert.equal(unknownAnthropicBlocks({ messages: "nope" }), null); + assert.equal(unknownAnthropicBlocks(body(txt(), imgUrl())), null); + assert.equal(unknownAnthropicBlocks(body({ role: "user", content: "plain string" } as never)), null); + + const r = unknownAnthropicBlocks({ + model: "claude-test", + messages: [ + { role: "assistant", content: [txt(), docBase64()] }, + { role: "user", content: [{ type: "custom_thing", v: 1 }] }, + { role: "user", content: [docUrl()] }, + ], + }); + assert.deepEqual(r, { count: 3, types: ["custom_thing", "document"], firstIndex: 0 }); +}); + +test("warnUnknownAnthropicBlocks logs once per session per distinct type-set", () => { + _resetWireDropWarningsForTest(); + const lines: string[] = []; + const lg = (_level: string, msg: string) => lines.push(msg); + + const d = body(docBase64()); + warnUnknownAnthropicBlocks(d, "ses_a", lg); + warnUnknownAnthropicBlocks(d, "ses_a", lg); // same session, same type-set → deduped + assert.equal(lines.length, 1); + assert.match(lines[0], /\[ses_a\]/); + assert.match(lines[0], /document/); + assert.match(lines[0], /#3410/); + + warnUnknownAnthropicBlocks(body({ type: "custom_thing", v: 1 }), "ses_a", lg); // new type-set → warns again + assert.equal(lines.length, 2); + warnUnknownAnthropicBlocks(d, "ses_b", lg); // new session → warns again + assert.equal(lines.length, 3); + warnUnknownAnthropicBlocks(body(txt(), imgUrl()), "ses_a", lg); // known types only → silent + assert.equal(lines.length, 3); + warnUnknownAnthropicBlocks(null, "ses_a", lg); // unparseable → silent + assert.equal(lines.length, 3); + + _resetWireDropWarningsForTest(); + warnUnknownAnthropicBlocks(d, "ses_a", lg); // after reset → warns again + assert.equal(lines.length, 4); +}); + +test("billing: anthropic document blocks are priced like the openai file precedent", () => { + const b64CostBytes = Math.ceil(PDF_B64.length / 4); + + // base64 document: sized from its payload, identical to an image carrying the same bytes + const docB64Cost = imageTokensInParsedBody("anthropic", body(docBase64())); + assert.equal(docB64Cost, b64CostBytes); + assert.equal(imageTokensInParsedBody("anthropic", body({ type: "image", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } })), b64CostBytes); + + // pixels mode: a PDF is not decodable as an image, so both fall back identically + const docPixels = imageTokensInParsedBody("anthropic", body(docBase64()), "pixels"); + assert.equal(docPixels, imageTokensInParsedBody("anthropic", body({ type: "image", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } }), "pixels")); + assert.ok(docPixels > 0); + + // url document: flat remote price + assert.equal(imageTokensInParsedBody("anthropic", body(docUrl())), REMOTE_IMAGE_TOKENS); + // bare/malformed-source document: flat remote price (never zero) + assert.equal(imageTokensInParsedBody("anthropic", body(docBare())), REMOTE_IMAGE_TOKENS); + + // mixed content sums + assert.equal(imageTokensInParsedBody("anthropic", body(txt(), docUrl(), imgUrl())), 2 * REMOTE_IMAGE_TOKENS); + + // regression anchors — image pricing is byte-for-byte what it was before #3410 + assert.equal(imageTokensInParsedBody("anthropic", body(imgUrl())), REMOTE_IMAGE_TOKENS); + assert.equal(imageTokensInParsedBody("anthropic", body(imgNoSource())), 0); + assert.equal(imageTokensInParsedBody("anthropic", body(txt())), 0); +}); + +test("billing: count walker stays in lockstep with the cost walker for every document shape", () => { + const shapes: Array<[string, unknown]> = [ + ["base64", docBase64()], + ["url", docUrl()], + ["bare", docBare()], + ["url image (anchor)", imgUrl()], + ["sourceless image (anchor)", imgNoSource()], + ["text (anchor)", txt()], + ]; + for (const [name, part] of shapes) { + const cost = imageTokensInParsedBody("anthropic", body(part)); + const count = countImagesInParsedBody("anthropic", body(part)); + assert.equal(count > 0, cost > 0, `${name}: cost/count sign mismatch (cost=${cost}, count=${count})`); + assert.ok(count <= 1, `${name}: expected at most one media part, got ${count}`); + } + assert.equal(countImagesInParsedBody("anthropic", body(docBase64(), docUrl(), docBare())), 3); +}); + +test("billing: the raw-body probe admits document-only bodies (compact and spaced JSON)", () => { + const compact = JSON.stringify(body(docBase64())); + assert.ok(compact.includes('"type":"document"')); + assert.equal(imageTokensInRawBody("anthropic", compact), Math.ceil(PDF_B64.length / 4)); + assert.equal(countImagesInRawBody("anthropic", compact), 1); + + const spaced = compact.replace(/"type":"/g, '"type": "'); + assert.ok(spaced.includes('"type": "document"')); + assert.equal(imageTokensInRawBody("anthropic", spaced), Math.ceil(PDF_B64.length / 4)); + + // clean bodies stay free — the probe must not over-admit + assert.equal(imageTokensInRawBody("anthropic", JSON.stringify(body(txt()))), 0); + assert.equal(countImagesInRawBody("anthropic", JSON.stringify(body(txt()))), 0); +});