Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.144",
"version": "0.0.146",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
21 changes: 20 additions & 1 deletion kernel/src/protected.ts
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,16 @@ export function hasMediaPayload(msg: CoreMessage): boolean {
if (Array.isArray(content))
return content.some((p) => !isObjWith(p, "type", "text"));
}
// #3410: unknown top-level block types (in the wild: document/PDF) ride the
// sidecar with their full payload — media like image, not text.
const abType = asObj(ab)?.type;
if (
typeof abType === "string" &&
abType !== "image" &&
abType !== "tool_result" &&
abType !== "redacted_thinking"
)
return true;
const item = m.rawResponsesItem;
if (isObjWith(item, "type", "input_image")) return true;
if (item && typeof item === "object") {
Expand Down Expand Up @@ -462,7 +472,16 @@ export function hasUnrecoverableMediaPayload(msg: CoreMessage): boolean {
}
}
}
// redacted_thinking etc. share this field — ignored, parity with hasMediaPayload
// #3410: unknown top-level block types (document/PDF ...) — the host
// archives no such payload to disk, so folding destroys it permanently.
if (
typeof ab.type === "string" &&
ab.type !== "image" &&
ab.type !== "tool_result" &&
ab.type !== "redacted_thinking"
)
return true;
// redacted_thinking shares this field — ignored, parity with hasMediaPayload
}

const item = asObj(m.rawResponsesItem);
Expand Down
20 changes: 20 additions & 0 deletions kernel/src/wire/anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,26 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat {
});
break;
}
default: {
// #3410: unknown top-level block types (in the wild today:
// document/PDF) used to fall out of this switch silently — the
// payload vanished on every rebuild with no warning. Ride the
// sidecar like image/redacted_thinking: coreToAnthropic re-emits
// rawAnthropicBlock verbatim (cache_control included) and never
// renders the label text, so the round trip is byte-exact.
const unknownType = (b as { type?: unknown }).type;
const label =
typeof unknownType === "string" ? `[${unknownType}]` : "[unknown]";
const base = deriveCanonicalMessageId(m.role, "text", label);
msgs.push({
id: clusters.next(base),
role: m.role,
contentType: "text",
text: label,
rawAnthropicBlock: b,
});
break;
}
}
}
for (let k = before; k < msgs.length; k++) {
Expand Down
14 changes: 13 additions & 1 deletion kernel/src/wire/google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,19 @@ export function normalizeGoogleUserPartOrder(parts: GooglePart[]): GooglePart[]
* becomes a single `{"role":"user"}` content normalized by
* normalizeGoogleUserPartOrder (#3159: plain text leads, functionResponse
* parts trail). Mid-conversation system cores (host-synthetic; Gemini has no
* system role inside contents) ride on the user side as a text part. */
* system role inside contents) ride on the user side as a text part.
*
* Byte-stability contract (#3938 / #3939⑥): for alternating-role histories
* (the only shape Gemini accepts), this function is PURE in the core array
* — same shape, same bytes, every request; any drift breaks the upstream
* prefix cache mid-session. The canonical part orders ARE the merge rules
* above: model runs emit reasoning → text → tool-call; user runs emit
* functionResponse parts then content parts, re-normalized by
* normalizeGoogleUserPartOrder (#3159). Merging same-role runs into one
* content is the required normalization (Gemini rejects non-alternating
* roles), and its output order is the canonical one. Do not reorder this
* emission without moving the pins in
* kernel/tests/issue3939-wire-byte-stability.test.ts. */
export function coreToGoogle(messages: BiliMessage[]): GoogleContent[] {
const out: GoogleContent[] = [];
let model: {
Expand Down
14 changes: 14 additions & 0 deletions kernel/src/wire/openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,20 @@ export function openaiToCore(body: OpenAIRequestBody): Flat {
return { msgs, systemText: systemParts.join("\n\n") };
}

/** Rebuild OpenAI Chat messages from kernel messages.
*
* Byte-stability contract (#3938 / #3939⑥): this function is PURE in the
* core array — the same logical shape must serialize to identical bytes on
* every request, because any drift breaks the upstream prefix cache
* mid-session. The canonical emission for a user run is FIXED: a lone
* plain-text run rebuilds as string content; any run carrying sidecars
* rebuilds as a parts array in CORE order where each sidecar core emits its
* text first, then its rawOpenaiContentParts in array order. The parse side
* (openaiToCore's leading-opaque PREPEND) makes that order stable for
* [opaque, text, opaque] shapes: every opaque part lands on one text core,
* so the rebuild always emits [text, opq1, opq2] regardless of the input
* interleaving. Do not reorder this emission without moving the pins in
* kernel/tests/issue3939-wire-byte-stability.test.ts. */
export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] {
const out: OpenAIMessage[] = [];
let userRun: BiliMessage[] = [];
Expand Down
143 changes: 143 additions & 0 deletions kernel/tests/issue3410-wire-passthrough.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { anthropicToCore, coreToAnthropic } from "../src/wire/anthropic.js";
import {
hasMediaPayload,
hasUnrecoverableMediaPayload,
} from "../src/protected.js";

// #3410: unknown top-level block types (in the wild today: document/PDF)
// fell out of anthropicToCore's switch silently — the payload vanished on
// every rebuild with no warning. The fix rides them on rawAnthropicBlock
// like image/redacted_thinking; these tests pin the round-trip fidelity and
// the fold protection that keeps the payload from being destroyed by a
// later compression.

const DOCUMENT = {
type: "document",
source: {
type: "base64",
media_type: "application/pdf",
data: "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlL0NhdGFsb2c+PgplbmRvYmoK",
},
cache_control: { type: "ephemeral" },
};

function userBody(content: object[]) {
return { messages: [{ role: "user", content }] };
}

function coreFields(m: { [k: string]: unknown }) {
return {
id: m.id,
role: m.role,
contentType: m.contentType,
text: m.text,
rawAnthropicBlock: m.rawAnthropicBlock,
};
}

test("round-trips an unknown document block byte-exact (#3410)", () => {
const body = userBody([{ type: "text", text: "summarize this pdf" }, DOCUMENT]);
const parsed = anthropicToCore(body as never);
assert.equal(parsed.msgs.length, 2);
const doc = parsed.msgs[1] as { [k: string]: unknown };
assert.equal(doc.contentType, "text");
assert.equal(doc.text, "[document]");
assert.deepEqual(doc.rawAnthropicBlock, DOCUMENT);

const rebuilt = coreToAnthropic(parsed.msgs as never);
assert.equal(rebuilt.length, 1);
// The label text never reaches the wire; the original block goes out
// verbatim, cache_control included.
assert.deepEqual(rebuilt[0], {
role: "user",
content: [{ type: "text", text: "summarize this pdf" }, DOCUMENT],
});
});

test("preserves block order when the unknown block leads", () => {
const body = userBody([DOCUMENT, { type: "text", text: "and this" }]);
const rebuilt = coreToAnthropic(anthropicToCore(body as never).msgs as never);
assert.deepEqual(rebuilt[0]!.content, [DOCUMENT, { type: "text", text: "and this" }]);
});

test("keeps multiple unknown block types verbatim", () => {
const future = { type: "future_block", foo: 1 };
const body = userBody([
{ type: "text", text: "t" },
DOCUMENT,
future,
]);
const rebuilt = coreToAnthropic(anthropicToCore(body as never).msgs as never);
assert.deepEqual(rebuilt[0]!.content, [
{ type: "text", text: "t" },
DOCUMENT,
future,
]);
});

test("preserves unknown blocks on assistant messages", () => {
const serverTool = { type: "server_tool_use", id: "st_1", name: "web_search" };
const body = { messages: [{ role: "assistant", content: [serverTool] }] };
const parsed = anthropicToCore(body as never);
assert.equal((parsed.msgs[0] as { [k: string]: unknown }).role, "assistant");
const rebuilt = coreToAnthropic(parsed.msgs as never);
assert.deepEqual(rebuilt[0], { role: "assistant", content: [serverTool] });
});

test("unknown-block cores are media AND unrecoverable — folding would destroy them", () => {
const parsed = anthropicToCore(userBody([DOCUMENT]) as never);
assert.equal(hasMediaPayload(parsed.msgs[0] as never), true);
assert.equal(hasUnrecoverableMediaPayload(parsed.msgs[0] as never), true);
});

test("known sidecar classifications stay unchanged (parity pins)", () => {
const base64Img = {
type: "image",
source: { type: "base64", media_type: "image/png", data: "QQ==" },
};
const urlImg = {
type: "image",
source: { type: "url", url: "https://example.com/x.png" },
};
const redacted = { type: "redacted_thinking", data: "REDACTED==" };
const cores = anthropicToCore(
userBody([base64Img, urlImg, redacted, { type: "text", text: "plain" }]) as never,
).msgs as { [k: string]: unknown }[];
// archivable base64 image: media yes, unrecoverable no (host spills to disk)
assert.equal(hasMediaPayload(cores[0] as never), true);
assert.equal(hasUnrecoverableMediaPayload(cores[0] as never), false);
// URL image: unrecoverable (no local archive)
assert.equal(hasMediaPayload(cores[1] as never), true);
assert.equal(hasUnrecoverableMediaPayload(cores[1] as never), true);
// redacted_thinking: existing parity — not media-classified here
assert.equal(hasMediaPayload(cores[2] as never), false);
assert.equal(hasUnrecoverableMediaPayload(cores[2] as never), false);
// plain text: neither
assert.equal(hasMediaPayload(cores[3] as never), false);
assert.equal(hasUnrecoverableMediaPayload(cores[3] as never), false);
});

test("ids are deterministic across repeated parses; duplicates stay distinct and stable", () => {
const body = userBody([DOCUMENT, { type: "text", text: "t" }, DOCUMENT]);
const once = anthropicToCore(body as never).msgs.map(coreFields);
const twice = anthropicToCore(body as never).msgs.map(coreFields);
assert.deepEqual(twice, once);
const docIds = [once[0]!.id, once[2]!.id] as string[];
assert.notEqual(docIds[0], docIds[1]);
});

test("parse is a structural fixed point over rebuild", () => {
const body = userBody([
{ type: "text", text: "before" },
DOCUMENT,
{ type: "text", text: "after" },
]);
const first = anthropicToCore(body as never).msgs.map(coreFields);
const rebuilt = coreToAnthropic(
anthropicToCore(body as never).msgs as never,
);
const second = anthropicToCore({ messages: rebuilt } as never).msgs.map(coreFields);
assert.deepEqual(second, first);
});
Loading
Loading