Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions src/loop/cache-control.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,101 @@ export function anthropicToolsCarryCacheControl(tools: unknown): boolean {
return tools.some((t) => typeof t === "object" && t !== null && (t as { cache_control?: unknown }).cache_control !== undefined);
}

/** Count the cache_control breakpoints on a tools array (WC-010's combined
* budget counts every one, not just whether any exist). */
export function countAnthropicToolsCacheControls(tools: unknown): number {
if (!Array.isArray(tools)) return 0;
let n = 0;
for (const t of tools) {
if (typeof t === "object" && t !== null && (t as { cache_control?: unknown }).cache_control !== undefined) n++;
}
return n;
}

/** Count the cache_control breakpoints on a top-level system field. */
export function countAnthropicSystemCacheControls(system: unknown): number {
if (!Array.isArray(system)) return 0;
let n = 0;
for (const b of system) {
if (typeof b === "object" && b !== null && (b as { cache_control?: unknown }).cache_control !== undefined) n++;
}
return n;
}

const CLIENT_MARK_IDS_KEY = "anthropicClientCacheMarks";

/** #6861: pin CLIENT-managed message breakpoints cumulatively — the same
* discipline as computeAnthropicMessageMarks, applied to the client's own
* marks. Claude Code stamps a cache_control breakpoint on its last message
* (+ last assistant message) and MOVES it as the conversation grows (#2188),
* and anthropicToCore harvests those controls into a per-request map that
* coreToAnthropic re-emits verbatim. Re-emitting only the current request's
* marks rewrites the old message's bytes on the next request (its breakpoint
* is gone) — a mid-history rewrite on EVERY request that breaks the byte
* prefix exactly there and collapses the cache to the system-only checkpoint
* (one #6861 request hit 8.7%). A marked message STAYS marked: once a client
* breakpoint is seen on a message, keep re-emitting it there in later
* requests, so the outbound history stays append-stable and the checkpoint
* at that position keeps matching. Pinned marks die only by folding (their
* message leaves the live view). The client's current marks are always emitted
* (they are the client's caching intent); pinning only adds the historical
* ones back, within Anthropic's 4-breakpoint budget (system + tools + message
* marks combined, WC-010). If the merged set would exceed the budget, the
* NEWEST pinned-only marks are evicted until it fits, so any seam lands next
* to the tail, never on an early element. Returns the merged map for
* coreToAnthropic (and for the round-2 adapter via anthropicClientCacheControls,
* #2499). */
export function pinClientMessageMarks(
clientControls: Map<string, unknown>,
liveIds: Set<string>,
session: { metadata: Record<string, unknown> },
messageCap: number,
): Map<string, unknown> {
const merged = new Map<string, unknown>(clientControls);
if (knobNoCacheControl()) {
delete session.metadata[CLIENT_MARK_IDS_KEY];
return merged;
}
const raw = session.metadata[CLIENT_MARK_IDS_KEY];
const pinned: { id: string; control: unknown }[] = [];
if (Array.isArray(raw)) {
for (const entry of raw) {
if (entry && typeof entry === "object" && typeof (entry as { id?: unknown }).id === "string" && (entry as { id: string }).id.length > 0) {
pinned.push({ id: (entry as { id: string }).id, control: (entry as { control?: unknown }).control });
}
}
}
// Retire marks whose messages folded away; keep the survivors' order.
let kept = pinned.filter((p) => liveIds.has(p.id));
// Pin the client's current marks (first-marked order) for later requests.
for (const [id, control] of clientControls) {
if (!liveIds.has(id)) continue;
if (kept.some((p) => p.id === id)) continue;
kept.push({ id, control });
}
for (const p of kept) {
if (!merged.has(p.id)) merged.set(p.id, p.control);
}
// WC-010 safety valve: the emitted set (pinned + current) must fit
// Anthropic's 4-breakpoint budget. The client's current marks are never
// dropped; evict the NEWEST pinned-only marks until it fits, so any
// seam lands next to the tail, never on an early element.
while (merged.size > messageCap) {
let evicted = false;
for (let i = kept.length - 1; i >= 0; i--) {
if (!clientControls.has(kept[i]!.id)) {
merged.delete(kept[i]!.id);
kept.splice(i, 1);
evicted = true;
break;
}
}
if (!evicted) break;
}
session.metadata[CLIENT_MARK_IDS_KEY] = kept;
return merged;
}

export function stampAnthropicSystemCacheControl(systemOut: AnthropicRequestBody["system"], ours = true): AnthropicRequestBody["system"] {
if (knobNoCacheControl()) return systemOut;
if (!ours) return systemOut;
Expand Down
26 changes: 19 additions & 7 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import { detectGenerationBoundary } from "../generation-boundary.js";
import { attachSubagentSessions } from "../subagent-sessions.js";
import { estimateCoreMessages, estimateCoreMessagesUpper, extractBillingAttributionBlock } from "../preflight.js";
import { dshNativeCompactionWitness, recordConflict } from "../conflict-watch.js";
import { anthropicToolsCarryCacheControl, computeAnthropicMessageMarks, stampAnthropicSystemCacheControl } from "../loop/cache-control.js";
import { anthropicToolsCarryCacheControl, computeAnthropicMessageMarks, countAnthropicSystemCacheControls, countAnthropicToolsCacheControls, pinClientMessageMarks, stampAnthropicSystemCacheControl } from "../loop/cache-control.js";
import { reconcileSystemAnchor } from "../system-anchor.js";
import { isStrictReasoningEcho, modelIdOf } from "../strict-echo.js";
import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js";
Expand Down Expand Up @@ -495,15 +495,27 @@ export async function prepareAnthropic(
anthropicCacheMarks = cacheControls.size > 0 || anthropicToolsCarryCacheControl(parsed.tools)
? undefined
: computeAnthropicMessageMarks(processedMessages as { id?: string }[], attachedRetrievals.length, session);
// #2499: hand the round-2 adapter the SAME marks source the steady path
// uses — `anthropicCacheMarks ?? cacheControls`. The client's harvested
// map is the fallback the rebuild applies when bili adds no marks of its
// own (a client-managed session), so its trigger-turn breakpoint survives.
clientCacheControls = cacheControls;
// #6861: a client-managed session's moving breakpoints are pinned
// cumulatively (same discipline as our own marks) so the outbound
// history stays append-stable: without pinning, the client's last-
// message breakpoint moves every turn and the old message's bytes
// rewrite mid-history on every request. The pinned map is what the
// round-2 adapter rebuilds from too (#2499). Proxy mode ONLY: in
// plugin mode the host owns the rolling marks (opencode retouch
// channel, "no accumulation" contract) — pinning there breaks the
// host's latest-only discipline (opencode-rolling-cache-marks).
const clientManaged = anthropicCacheMarks === undefined && !pluginMode;
if (clientManaged) {
const messageCap = Math.max(0, 4 - countAnthropicSystemCacheControls(parsed.system) - countAnthropicToolsCacheControls(parsed.tools));
const liveIds = new Set(processedMessages.map((m) => m.id).filter((id): id is string => typeof id === "string" && id.length > 0));
clientCacheControls = pinClientMessageMarks(cacheControls, liveIds, session, messageCap);
} else {
clientCacheControls = cacheControls;
}
// #3400: egress image rescale — outbound copies only; the originals
// stay in session state/persistence untouched.
const egressMessages = await applyImageRescalePass(processedMessages as BiliMessage[], { session, sessionId, log, maxDim: imageRescaleMaxDim(), toolArmed: imageRescaleToolArmed(session) });
rebuiltMessages = coreToAnthropic(egressMessages, anthropicCacheMarks ?? cacheControls);
rebuiltMessages = coreToAnthropic(egressMessages, anthropicCacheMarks ?? clientCacheControls);
if (sysNotes.length > 0) {
rebuiltMessages = insertBeforeTrailingSystemRun(rebuiltMessages, sysNotes.map((text) => ({ role: "user" as const, content: text })));
}
Expand Down
240 changes: 240 additions & 0 deletions tests/issue6861-client-cache-marks-pinned.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
import assert from "node:assert";
import http from "node:http";
import { once } from "node:events";
import test from "node:test";

process.env.NODE_ENV = "test";

import { defaultConfig } from "acp-kernel";
import { startServer } from "../src/server.ts";
import type { ProxyOptions } from "../src/config.ts";
import { pinClientMessageMarks } from "../src/loop/cache-control.ts";

// #6861: `bili claude` rewrites mid-history on every request, collapsing the
// prefix cache (one request hit 8.7%, 119K tokens re-billed). Claude Code
// stamps a cache_control breakpoint on its last message (+ last assistant
// message) and MOVES it as the conversation grows (#2188); anthropicToCore
// harvests those controls into a per-request map that coreToAnthropic
// re-emits verbatim. Re-emitting only the current request's marks strips
// the breakpoint off the message that carried it last turn, so that
// message's bytes rewrite mid-history on the next request — a seam on EVERY
// request with no attribution (the seam detector fingerprints core messages,
// where the marks do not live). The fix pins client-managed breakpoints
// cumulatively (same discipline as bili's own marks, #1637): a marked
// message STAYS marked until it folds away. Proxy mode only — in plugin
// mode the host owns the rolling marks (opencode retouch channel, "no
// accumulation" contract); pinning there breaks the host's latest-only
// discipline (opencode-rolling-cache-marks).

const EPHEMERAL = { type: "ephemeral" } as const;

function session() {
return { metadata: {} as Record<string, unknown> };
}

test("#6861 unit: a turn-1 client mark stays pinned when the client moves it on turn 2", () => {
const s = session();
const turn1 = pinClientMessageMarks(new Map([["msg-a", EPHEMERAL]]), new Set(["msg-a"]), s, 3);
assert.equal(turn1.size, 1, "turn 1: only the client's own mark");
assert.deepEqual(s.metadata["anthropicClientCacheMarks"], [{ id: "msg-a", control: EPHEMERAL }], "turn 1 mark recorded");

// Turn 2: the client moved its breakpoint to msg-b; msg-a must stay marked.
const turn2 = pinClientMessageMarks(new Map([["msg-b", EPHEMERAL]]), new Set(["msg-a", "msg-b"]), s, 3);
assert.equal(turn2.size, 2, "turn 2: current + pinned");
assert.deepEqual(turn2.get("msg-a"), EPHEMERAL, "turn-1 mark pinned back");
assert.deepEqual(turn2.get("msg-b"), EPHEMERAL, "current mark always emitted");
});

test("#6861 unit: a pinned mark retires when its message folds away", () => {
const s = session();
pinClientMessageMarks(new Map([["msg-a", EPHEMERAL]]), new Set(["msg-a"]), s, 3);
const after = pinClientMessageMarks(new Map([["msg-b", EPHEMERAL]]), new Set(["msg-b"]), s, 3);
assert.equal(after.has("msg-a"), false, "folded message's mark retired");
assert.equal(after.size, 1, "only the live mark remains");
});

test("#6861 unit: over budget, the NEWEST pinned-only marks are evicted first; current marks never drop", () => {
const s = session();
// Cap 1 with two live pinned marks and one current mark: the newest
// pinned-only mark goes first, so any seam lands next to the tail.
pinClientMessageMarks(new Map([["msg-a", EPHEMERAL]]), new Set(["msg-a", "msg-b", "msg-c"]), s, 3);
pinClientMessageMarks(new Map([["msg-b", EPHEMERAL]]), new Set(["msg-a", "msg-b", "msg-c"]), s, 3);
const merged = pinClientMessageMarks(new Map([["msg-c", EPHEMERAL]]), new Set(["msg-a", "msg-b", "msg-c"]), s, 1);
assert.equal(merged.has("msg-c"), true, "current mark never dropped");
assert.equal(merged.has("msg-a"), false, "newest pinned-only mark evicted first");
assert.equal(merged.size, 1, "merged set fits the cap");
});

test("#6861 unit: BILI_NO_CACHE_CONTROL=1 clears stored pins and returns only current marks", () => {
const prev = process.env.BILI_NO_CACHE_CONTROL;
process.env.BILI_NO_CACHE_CONTROL = "1";
try {
const s = session();
pinClientMessageMarks(new Map([["msg-a", EPHEMERAL]]), new Set(["msg-a"]), s, 3);
const merged = pinClientMessageMarks(new Map([["msg-b", EPHEMERAL]]), new Set(["msg-a", "msg-b"]), s, 3);
assert.equal(merged.has("msg-a"), false, "no pinning while disabled");
assert.equal(merged.size, 1, "only the current mark");
assert.equal(s.metadata["anthropicClientCacheMarks"], undefined, "stored pins cleared");
} finally {
if (prev === undefined) delete process.env.BILI_NO_CACHE_CONTROL;
else process.env.BILI_NO_CACHE_CONTROL = prev;
}
});

function listen(server: http.Server): Promise<void> {
if (server.listening) return Promise.resolve();
return once(server, "listening").then(() => undefined);
}

function close(server: http.Server): Promise<void> {
const { promise, resolve, reject } = Promise.withResolvers<void>();
server.close((error) => (error ? reject(error) : resolve()));
return promise;
}

function anthropicSse(event: string, data: unknown): string {
return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
}

function textScript(): string {
return anthropicSse("message_start", { type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: 42 } } }) +
anthropicSse("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }) +
anthropicSse("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }) +
anthropicSse("content_block_stop", { type: "content_block_stop", index: 0 }) +
anthropicSse("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } }) +
anthropicSse("message_stop", { type: "message_stop" });
}

interface Rig {
proxyPort: number;
portHint(): number;
upstreamBodies: string[];
closeAll(): Promise<void>;
}

async function startRig(): Promise<Rig> {
const upstreamBodies: string[] = [];
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
upstreamBodies.push(Buffer.concat(chunks).toString("utf8"));
res.writeHead(200, { "content-type": "text/event-stream" });
res.end(textScript());
});
});
upstream.listen(0, "127.0.0.1");
await listen(upstream);
const upstreamPort = (upstream.address() as { port: number }).port;

const opts: ProxyOptions = {
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "l6861-model": { context: 100_000 } } } },
modelContextLimit: 100_000,
kernelConfig: defaultConfig(100_000),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
compat: { roles: {} },
streamErrorShape: "protocol",
passthroughSource: null,
autoRestartOnUpdate: false,
updateTag: "latest",
advisoryCheck: false,
releaseNotesCheck: false,
log: false,
sessionHeader: "x-acp-session",
debug: false,
passthrough: false,
autoUpdate: false,
mitm: { enabled: false, domains: [] },
};
let server: http.Server;
try {
server = await startServer(opts);
} catch (error) {
await close(upstream);
throw error;
}
await listen(server);
const proxyPort = (server.address() as { port: number }).port;
return {
proxyPort,
portHint: () => upstreamPort,
upstreamBodies,
closeAll: async () => {
await close(server);
await close(upstream);
},
};
}

interface Msg {
role: "user" | "assistant";
content: string | { type: "text"; text: string; cache_control?: unknown }[];
}

async function postModel(rig: Rig, sessionId: string, messages: Msg[]): Promise<Response> {
return fetch(`http://127.0.0.1:${rig.proxyPort}/bili/http://127.0.0.1:${rig.portHint()}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-session-affinity": sessionId },
body: JSON.stringify({ model: "l6861-model", max_tokens: 8192, stream: true, messages }),
});
}

async function waitFor(pred: () => boolean, what: string): Promise<void> {
for (let i = 0; i < 200; i++) {
if (pred()) return;
await new Promise((r) => setTimeout(r, 25));
}
assert.ok(pred(), `timeout waiting for ${what}`);
}

interface AnthropicBody {
messages: { role: string; content: string | { type: string; text?: string; cache_control?: unknown }[] }[];
}

function markedIndexes(body: string): number[] {
const parsed = JSON.parse(body) as AnthropicBody;
const out: number[] = [];
parsed.messages.forEach((m, i) => {
if (Array.isArray(m.content) && m.content.some((b) => b.cache_control)) out.push(i);
});
return out;
}

test("#6861 e2e: the client's moving breakpoint stays pinned, so turn 1's messages serialize byte-identically on turn 2", async () => {
const rig = await startRig();
try {
const sid = `sess-6861-${process.pid}`;
// Turn 1: the client marks its last message (u1), as Claude Code does.
const h1: Msg[] = [
{ role: "user", content: [{ type: "text", text: "u1", cache_control: { type: "ephemeral" } }] },
{ role: "assistant", content: "a1" },
];
await postModel(rig, sid, h1);
await waitFor(() => rig.upstreamBodies.length >= 1, "turn 1 forward");
assert.deepEqual(markedIndexes(rig.upstreamBodies[0]), [0], "turn 1: only the client's mark");

// Turn 2: the client moved its breakpoint to the new last message (u2)
// and sends u1 unmarked. Without pinning, u1's bytes rewrite
// mid-history; with pinning, u1 keeps its breakpoint and the
// turn-1 prefix stays byte-stable.
const h2: Msg[] = [
{ role: "user", content: "u1" },
{ role: "assistant", content: "a1" },
{ role: "user", content: [{ type: "text", text: "u2", cache_control: { type: "ephemeral" } }] },
];
await postModel(rig, sid, h2);
await waitFor(() => rig.upstreamBodies.length >= 2, "turn 2 forward");

const b1 = JSON.parse(rig.upstreamBodies[0]) as AnthropicBody;
const b2 = JSON.parse(rig.upstreamBodies[1]) as AnthropicBody;
assert.deepEqual(markedIndexes(rig.upstreamBodies[1]), [0, 2], "turn 2: pinned turn-1 mark + current mark");
assert.deepEqual(b2.messages[0], b1.messages[0], "turn-1 marked message byte-stable across turns (prefix-cache contract)");
assert.deepEqual(b2.messages[1], b1.messages[1], "turn-1 tail message byte-stable across turns");
} finally {
await rig.closeAll();
}
});
Loading