Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 80 additions & 19 deletions src/auto-mode-forge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,26 @@
// block flavor → <block>no</block>
// (begins with <block>, single agreeing occurrence, no → allow)
//
// Detection: the magic stop sequences are the strongest signal but ONLY
// stage-1 non-fast carries them — stage-1-fast and stage-2 have none. The
// second signal is structural: the flattened conversation rides in a
// <transcript>…</transcript> wrapper in the user message(s), and the request
// carries the classifier's policy wording (product copy, stable across
// versions). Either signal alone can appear in legitimate traffic; the
// conjunction cannot reasonably be faked by a coding session. If detection
// fires but the flavor is undecidable, we do NOT forge — we fall back to
// verbatim passthrough (never emit a malformed verdict, #353 discipline).
// Detection: three-part gate, biased toward FALSE NEGATIVES — a miss degrades
// to raw forwarding (pre-#2740 behavior: billed upstream, safe); a hit either
// forges an allow verdict on a real turn or kills the session's compression
// silently (#3908: a tool result quoting this very file carried both marker
// families and poisoned every later request of that session).
// 1. STRUCTURE — a well-formed <transcript>…</transcript> pair in the USER
// TEXT enclosing >= MIN_TRANSCRIPT_SPAN chars: the wrapper carries the
// ENTIRE flattened session (KB..MB scale from the first classified-tool
// exchange on); quoted mentions (source literals, doc examples, comment
// spans) enclose at most ~100 chars.
// 2. PLACEMENT — the corroborating signal must sit where CC actually puts
// it: the magic stop_sequences (top-level field) OR policy wording
// (product copy, stable across versions) in SYSTEM or the same user text
// blocks. tool_result payloads / assistant text / tools JSON never count
// — the one-shot carries no tool results, and scanning executed output is
// what made #3908 possible.
// Magic stops are the strongest signal but ONLY stage-1 non-fast carries them;
// stage-1-fast and stage-2 have none, so the policy-wording arm covers those.
// If detection fires but the flavor is undecidable, we do NOT forge — verbatim
// passthrough (never emit a malformed verdict, #353 discipline).

import type { AnthropicRequestBody } from "acp-kernel/wire";
import { extractSystem } from "acp-kernel/wire";
Expand All @@ -96,6 +107,14 @@ const SHARED_POLICY_MARKERS = [
"explicit (not suggestive or implicit) user confirmation is required to override blocks",
];
const TRANSCRIPT_MARKER = "<transcript>";
const TRANSCRIPT_CLOSE = "</transcript>";
/** Smallest span a genuine wrapper encloses. The flattened transcript starts
* KB-scale at the first classified-tool exchange (CC's jsonl envelope per
* entry); the largest quoted-artifact span observed in the #3908 class
* (source literal / comment "…" span) is under ~100 chars. 512 sits between
* with >=5x margin per side; a miss below the floor forwards raw upstream —
* the safe direction (see the Detection block above). */
const MIN_TRANSCRIPT_SPAN = 512;

function messageTextOf(m: { role?: string; content?: unknown }): string {
const c = m.content;
Expand Down Expand Up @@ -128,22 +147,64 @@ export function requestText(parsed: AnthropicRequestBody): string {
return parts.join("\n");
}

/** True for BOTH stages and BOTH flavors — see the module header for why each
* signal exists and which shapes carry it. The <transcript> wrapper is the
* load-bearing structural signal (every stage, every flavor carries it); the
* magic stop sequences corroborate but are NOT sufficient alone — "</severity>"
* / "</block>" are ordinary XML tags a legitimate session can legitimately use
* as stop hints (any harness emitting severity/block verdict XML). Detecting on
* stops alone would forge a local "always allow" verdict for such traffic. */
/** Text of one message as far as the classifier FRAME is concerned: plain
* user text blocks only. The one-shot request carries no assistant turns and
* no tool results, so executed output must never count toward detection —
* a tool_result quoting marker-bearing source poisoned whole sessions
* (#3908). */
function frameTextOf(m: { role?: string; content?: unknown }): string {
if (m.role !== "user") return "";
const c = m.content;
if (typeof c === "string") return c;
if (!Array.isArray(c)) return "";
let out = "";
for (const b of c) {
if (b === null || typeof b !== "object") continue;
const rec = b as Record<string, unknown>;
if (rec.type === "tool_result") continue;
if (typeof rec.text === "string") out += rec.text;
}
return out;
}

/** Largest char span enclosed by any well-formed <transcript>…</transcript>
* pair (greedy left-to-right pairing, max over all pairs). Bare mentions
* without a close tag score 0. */
function maxTranscriptSpan(text: string): number {
let best = 0;
let from = 0;
for (;;) {
const open = text.indexOf(TRANSCRIPT_MARKER, from);
if (open < 0) break;
const close = text.indexOf(TRANSCRIPT_CLOSE, open + TRANSCRIPT_MARKER.length);
if (close < 0) break;
const span = close - open - TRANSCRIPT_MARKER.length;
if (span > best) best = span;
from = close + TRANSCRIPT_CLOSE.length;
}
return best;
}

/** True for BOTH stages and BOTH flavors — see the module header's Detection
* block for why each gate exists. Order matters: the substantial wrapper is
* checked FIRST because it is the load-bearing structural signal (every stage,
* every flavor carries one, and a genuine wrapper encloses the entire
* flattened session); magic stops corroborate but are NOT sufficient alone —
* "</severity>" / "</block>" are ordinary XML tags a legitimate session can
* legitimately use as stop hints (any harness emitting severity/block verdict
* XML). Policy wording is searched in the FRAME only (system + user text
* blocks): scanning tool_result/assistant/tools history for it was the #3908
* poison channel. */
export function isAutoModeClassifier(parsed: AnthropicRequestBody): boolean {
const text = requestText(parsed);
if (!text.includes(TRANSCRIPT_MARKER)) return false;
const userText = (parsed.messages ?? []).map(frameTextOf).join("\n");
if (maxTranscriptSpan(userText) < MIN_TRANSCRIPT_SPAN) return false;
const stops = parsed.stop_sequences;
const hasMagicStops =
Array.isArray(stops) && stops.some((s) => typeof s === "string" && AUTO_MODE_CLASSIFIER_STOPS.has(s));
if (hasMagicStops) return true; // wrapper + magic stops: stage-1 non-fast, even if policy copy drifts
const frame = extractSystem(parsed.system) + "\n" + userText;
return [...SEVERITY_FLAVOR_MARKERS, ...BLOCK_FLAVOR_MARKERS, ...SHARED_POLICY_MARKERS]
.some((marker) => text.includes(marker));
.some((marker) => frame.includes(marker));
}

/** Which forged verdict shape this request expects. null = undecidable →
Expand Down
59 changes: 54 additions & 5 deletions tests/auto-mode-forge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ test("estimateCjkTokens: #2740 incident calibration — 2,669,226 chars billed 1

const SEVERITY_SYS = "You are a safety classifier. 50 is exactly the allow/block boundary.";
const BLOCK_SYS = 'Respond with <block>yes</block> or <block>no</block>.';
const TRANSCRIPT_USER = [{ type: "text" as const, text: "<transcript>\n[user] run deploy\n[assistant] done\n</transcript>" }];
// Realistic scale: the wrapper carries the ENTIRE flattened session — even the
// smallest genuine one (a single classified-tool exchange rendered from CC's
// jsonl envelope) is KB-scale. Sub-KB fixtures would paper over the #3908 span
// floor and let quoted-artifact shapes through undetected.
const BIG_TRANSCRIPT = "<transcript>\n" + "[user] run deploy\n[assistant] ok\n".repeat(40) + "</transcript>";
const TRANSCRIPT_USER = [{ type: "text" as const, text: BIG_TRANSCRIPT }];

test("isAutoModeClassifier: stage-1 non-fast — magic stop sequences corroborated by the <transcript> wrapper", () => {
assert.ok(isAutoModeClassifier(req({ stop_sequences: ["</severity>"], messages: [{ role: "user", content: TRANSCRIPT_USER }] })));
Expand Down Expand Up @@ -81,18 +86,23 @@ test("isAutoModeClassifier: no-stop shapes (stage-1-fast / stage-2) — policy w
assert.ok(isAutoModeClassifier(req({
system: "You are a safety classifier.",
messages: [{ role: "user", content: [
{ type: "text", text: "<transcript>…</transcript>" },
{ type: "text", text: BIG_TRANSCRIPT },
{ type: "text", text: "Err on the side of blocking when uncertain." },
] }],
})));
});

test("isAutoModeClassifier: marker inside a tool_result payload counts (messageTextOf walk)", () => {
assert.ok(isAutoModeClassifier(req({
test("isAutoModeClassifier: policy wording in a tool_result payload does NOT count (#3908)", () => {
// Inverted from the pre-#3908 assertion: the one-shot classifier request
// carries no tool results, so executed output is outside the frame. Letting
// it count is exactly how a source snippet poisoned a whole session. The
// wrapper is a genuine-scale one so this pins PLACEMENT alone, independent
// of the span floor.
assert.ok(!isAutoModeClassifier(req({
system: "You are a safety classifier.",
messages: [{ role: "user", content: [
{ type: "tool_result", tool_use_id: "t1", content: "Review the classification process before finalizing." },
{ type: "text", text: "<transcript>…</transcript>" },
{ type: "text", text: BIG_TRANSCRIPT },
] }],
})));
});
Expand All @@ -105,6 +115,45 @@ test("isAutoModeClassifier: negatives — each signal alone is not enough", () =
assert.ok(!isAutoModeClassifier(req({ stop_sequences: ["</severity>"] })), "magic stop sequence without the transcript wrapper");
});

test("isAutoModeClassifier: #3908 regression — quoted source snippet in a tool_result no longer poisons the session", () => {
// The incident shape: a multi-turn session whose tool result carried
// src/auto-mode-forge.ts itself — "<transcript>" literal, the paired
// comment span, and "Err on the side of blocking" all in ONE payload. Every
// later request resends that history and matched forever.
const sourceSnippet = [
"// the flattened conversation rides in a <transcript>…</transcript> wrapper",
'const TRANSCRIPT_MARKER = "<transcript>";',
'const SHARED_POLICY_MARKERS = ["Err on the side of blocking"];',
].join("\n");
const msgs: Array<{ role: string; content: unknown }> = [];
for (let i = 0; i < 30; i++) {
msgs.push({ role: "user", content: `turn ${i} question` });
msgs.push({ role: "assistant", content: `turn ${i} answer` });
}
msgs.push({ role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: sourceSnippet }] });
msgs.push({ role: "assistant", content: "continuing the task" });
assert.ok(!isAutoModeClassifier(req({ messages: msgs as AnthropicRequestBody["messages"] })), "poisoned multi-turn session");
});

test("isAutoModeClassifier: #3908 regression — sub-floor paired span in plain user text is not the wrapper", () => {
// Chat-paste variant of the same defect: both marker families in typed
// text, but the paired span is a comment-style ellipsis (1 char), far under
// the MIN_TRANSCRIPT_SPAN floor.
assert.ok(!isAutoModeClassifier(req({
messages: [{ role: "user", content: "look:\n<transcript>…</transcript>\nand remember: Err on the side of blocking." }],
})));
});

test("isAutoModeClassifier: magic stops still require a SUBSTANTIAL wrapper", () => {
// Pre-#3908, stops + a bare mention matched. A genuine stage-1 non-fast
// always wraps the full session, so sub-floor spans stay negative even with
// magic stops present.
assert.ok(!isAutoModeClassifier(req({
stop_sequences: ["</severity>"],
messages: [{ role: "user", content: "<transcript>…</transcript> please classify" }],
})));
});

// --- variant resolution ---

test("classifierForgeVariant: stop sequences win over body markers", () => {
Expand Down
12 changes: 8 additions & 4 deletions tests/e2e-anthropic.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ const UPSTREAM_SYSTEM = "You are a test assistant.";
// ~3660 chars of padding: the kernel refuses to fold ranges below a 5000-char
// minimum, so the state-equivalence fixture's compressed range needs mass.
const FILLER = "lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(60);
// A genuine CC auto-mode one-shot carries the ENTIRE flattened session inside
// the <transcript> wrapper (KB..MB scale); sub-KB placeholders fall below the
// #3908 span floor and must NOT be detected as classifier requests.
const CLASSIFIER_TRANSCRIPT = "<transcript>\n" + "[user] run deploy\n[assistant] ok\n".repeat(40) + "</transcript>";

function anthropicSse(event: string, data: unknown): string {
return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
Expand Down Expand Up @@ -443,7 +447,7 @@ test("e2e anthropic: auto-mode classifier forged locally by default — zero ups
max_tokens: 64,
stream: true,
system: "You are a safety classifier.",
messages: [{ role: "user", content: "<transcript>\n[user] Is this command safe? rm -rf /\n</transcript>" }],
messages: [{ role: "user", content: CLASSIFIER_TRANSCRIPT }],
stop_sequences: ["</severity>"],
});
assert.equal(h.captured.length, 0, `classifier request must produce ZERO upstream egress, got ${h.captured.length} requests`);
Expand All @@ -468,7 +472,7 @@ test("e2e anthropic: auto-mode classifier block flavor forged as bare <block>no<
max_tokens: 64,
stream: true,
system: "You are a safety classifier.",
messages: [{ role: "user", content: "<transcript>\n[user] Is this command safe? rm -rf /\n</transcript>" }],
messages: [{ role: "user", content: CLASSIFIER_TRANSCRIPT }],
stop_sequences: ["</block>"],
});
assert.equal(h.captured.length, 0, "zero upstream egress");
Expand Down Expand Up @@ -510,7 +514,7 @@ test("e2e anthropic: auto-mode classifier no-stop shape (stage-2/fast) forged vi
max_tokens: 8192,
stream: true,
system: "You are a safety classifier. 50 is exactly the allow/block boundary.",
messages: [{ role: "user", content: "<transcript>\n[user] deploy to production\n</transcript>" }],
messages: [{ role: "user", content: CLASSIFIER_TRANSCRIPT }],
});
assert.equal(h.captured.length, 0, "no-stop classifier shapes must also be forged locally");
const events = parseAnthropicSse(raw);
Expand Down Expand Up @@ -557,7 +561,7 @@ test("e2e anthropic: forged classifier call leaves ACP state byte-equivalent (#2
max_tokens: 64,
stream: true,
system: "You are a safety classifier.",
messages: [{ role: "user", content: "<transcript>\n[user] rm -rf /\n</transcript>" }],
messages: [{ role: "user", content: CLASSIFIER_TRANSCRIPT }],
stop_sequences: ["</severity>"],
});
assert.equal(h.captured.length, preForgeCalls, "forged classifier call must add zero upstream egress");
Expand Down
Loading