From f7a247f40a28ef25954b68f1492e4c5058bd19df Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 12 Oct 2026 02:44:20 +0800 Subject: [PATCH] fix: preflight classifies normal-stop zero-text summaries as deterministic (#2832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - diagnoseEmptySummary gains per-wire block fingerprints (text_chars / text_blocks / thinking_blocks / reasoning_chars / text_parts) so a thinking-only answer is distinguishable from relay-stripped text in logs and fail-fast details - new isDeterministicEmptyCompletion: a normal stop (end_turn / stop / completed) carrying zero answer text is deterministic for that (model, host) pair — it earns neither #1767's same-span retries nor #726's halving recovery, because re-sending the identical payload returns the identical empty answer and a normal stop is not a size signal - three consecutive deterministic empties on the same (host, model) arm a session-persisted dead-probe verdict (session.metadata deterministicEmptyProbe); while dead, at most ONE canary probe per preflight invocation bills the wall, and a successful canary clears the verdict so folding resumes on a self-healed relay - fail-fast detail names the actionable exits: configure compress.externalSummary, switch the session's model, or fix the relay's thinking/text handling - tests: classifier matrix + fingerprint pins (unit), systemic e2e (bounded strikes, sticky verdict, exactly-one-canary contract, nothing forwarded), recovery e2e (canary clears, fold proceeds, 200); #574 multi-range fixture moves to max_tokens so that file keeps pinning the walk/budget mechanics on the size-driven path independent of this reclassification --- src/preflight.ts | 217 +++++++++++- ...2832-preflight-deterministic-empty.test.ts | 323 ++++++++++++++++++ tests/preflight-empty-summary.test.ts | 7 +- tests/preflight-multi-range.test.ts | 10 +- tests/preflight-transient-empty-retry.test.ts | 4 +- 5 files changed, 552 insertions(+), 9 deletions(-) create mode 100644 tests/issue2832-preflight-deterministic-empty.test.ts diff --git a/src/preflight.ts b/src/preflight.ts index f640423bce..55929d1653 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -90,6 +90,17 @@ const TRANSIENT_EMPTY_SUMMARY_RETRIES = 2; // few extra calls instead of burning the full budget on doomed draws. // Reset alongside summaryCalls when soft protection is relaxed (#575-merge). const TRANSIENT_EMPTY_RETRY_BUDGET = 4; +// #2832: a completion that ends NORMALLY (end_turn / stop / completed) with +// ZERO answer text is not a blip — the model spent its whole turn in the +// thinking channel and stopped cleanly, or the relay stripped the text +// blocks. Re-sending the identical payload returns the identical empty +// answer, so unlike #1767's transient empties (content_filter, truncated +// streams) it earns no same-span retry, and unlike #726's size-driven empties +// halving the span cannot help either: a normal stop is not a size signal. +// K consecutive such answers on the same (host, model) arm a session-level +// dead-probe verdict that stops further probing for this session until a +// probe succeeds again (canary probe per invocation — see preflightCompress). +const DETERMINISTIC_EMPTY_STICKY_LIMIT = 3; // #1841: slack on the futility verdicts below. Range token estimates carry // ±~20% error on mixed CJK/Latin/code, and a fold's summary re-enters the // payload, so realizable saving is strictly below span mass. Fail CLOSED: @@ -237,7 +248,7 @@ interface PreflightFailure { // unusable branch carries a diagnosis of what the body actually contained so // it is logged and surfaced in the fail-fast message instead of the generic // "summary too short". -type SummaryOutcome = { summary: string; external?: boolean } | { unusable: string; transient?: boolean; external?: never }; +type SummaryOutcome = { summary: string; external?: boolean } | { unusable: string; transient?: boolean; deterministic?: boolean; external?: never }; export interface PreflightResult { compressedRanges: number; @@ -826,6 +837,41 @@ export function emptySummaryIsSizeDriven(diagnosis: string): boolean { return /context_length_exceeded|too long|finish_reason=length\b|stop_reason=max_tokens\b|reason=max_output_tokens\b/i.test(diagnosis); } +// #2832: a plain-JSON completion that ended NORMALLY (end_turn / stop / +// completed) yet carries ZERO answer text is deterministic for this +// (model, host) pair — the model spent its whole turn in the thinking +// channel and stopped cleanly, or the relay dropped the text blocks. +// Re-sending the identical payload yields the identical empty answer, so it +// earns neither #1767's same-span retries nor #726's halving recovery (a +// normal stop is not a size signal). SSE bodies (json === null) are never +// classified here — truncation mid-stream stays the transient case #1767/#780 +// defend. A non-zero answer under MIN_SUMMARY_CHARS also stays off this path: +// only a genuinely EMPTY normal stop proves the probe path itself is dead. +export function isDeterministicEmptyCompletion(protocol: PreflightProtocol, json: unknown): boolean { + if (!json || typeof json !== "object") return false; + if (extractSummaryText(protocol, json as Record).trim().length > 0) return false; + const o = json as Record; + switch (protocol) { + case "anthropic": { + return o.stop_reason === "end_turn" && Array.isArray(o.content); + } + case "openai": { + const choices = o.choices; + if (!Array.isArray(choices) || choices.length === 0) return false; + const first = choices[0]; + if (!first || typeof first !== "object") return false; + return (first as Record).finish_reason === "stop"; + } + case "responses": { + if (!Array.isArray(o.output)) return false; + if (typeof o.status === "string" && o.status !== "completed") return false; + return true; + } + default: + return false; + } +} + export function diagnoseEmptySummary(text: string, json?: unknown): string { if (json && typeof json === "object") { // #2309: the Responses terminal check runs FIRST — a failed response @@ -887,12 +933,79 @@ function emptyCompletionDetail(json: Record): string | null { if (Array.isArray(choices) && choices.length > 0) { const first = choices[0]; if (!first || typeof first !== "object") return null; - const fr = (first as Record).finish_reason; + const choice = first as Record; + const fr = choice.finish_reason; if (typeof fr === "string") parts.push(`finish_reason=${fr}`); + // #2832: block-level fingerprint — an EMPTY answer with a NORMAL stop + // reason is only diagnosable when the log shows where the model's + // output went (thinking-only vs relay-stripped text vs truly nothing). + const message = choice.message; + if (message && typeof message === "object") { + const msg = message as Record; + if (typeof msg.content === "string") { + parts.push(`text_chars=${msg.content.length}`); + } else if (Array.isArray(msg.content)) { + let textBlocks = 0; + for (const part of msg.content) { + if (part && typeof part === "object" && typeof (part as Record).text === "string") textBlocks += 1; + } + parts.push(`text_blocks=${textBlocks}`); + } + if (typeof msg.reasoning_content === "string" && msg.reasoning_content.length > 0) { + parts.push(`reasoning_chars=${msg.reasoning_content.length}`); + } + } } else if (typeof json.stop_reason === "string") { parts.push(`stop_reason=${json.stop_reason}`); + if (Array.isArray(json.content)) { + let textBlocks = 0; + let thinkingBlocks = 0; + let reasoningChars = 0; + for (const block of json.content) { + if (!block || typeof block !== "object") continue; + const b = block as Record; + if (b.type === "text") textBlocks += 1; + else if (b.type === "thinking" || b.type === "redacted_thinking") { + thinkingBlocks += 1; + if (typeof b.thinking === "string") reasoningChars += b.thinking.length; + } + } + parts.push(`text_blocks=${textBlocks}`); + if (thinkingBlocks > 0) parts.push(`thinking_blocks=${thinkingBlocks}`); + if (reasoningChars > 0) parts.push(`reasoning_chars=${reasoningChars}`); + } } else if (Array.isArray(json.output)) { if (typeof json.status === "string") parts.push(`status=${json.status}`); + // #2832 responses shape: mirror extractSummaryText's counting — text + // parts only from message items, reasoning chars from reasoning items. + let textParts = 0; + let reasoningChars = 0; + for (const item of json.output) { + if (!item || typeof item !== "object") continue; + const it = item as Record; + const itemType = typeof it.type === "string" ? it.type : ""; + if (itemType === "reasoning") { + for (const arr of [it.content, it.summary]) { + if (!Array.isArray(arr)) continue; + for (const p of arr) { + if (p && typeof p === "object" && typeof (p as Record).text === "string") { + reasoningChars += ((p as Record).text as string).length; + } + } + } + } else if (itemType === "" || itemType === "message") { + const content = it.content; + if (!Array.isArray(content)) continue; + for (const p of content) { + if (!p || typeof p !== "object") continue; + const part = p as Record; + const partType = typeof part.type === "string" ? part.type : ""; + if ((partType === "" || partType === "output_text") && typeof part.text === "string") textParts += 1; + } + } + } + parts.push(`text_parts=${textParts}`); + if (reasoningChars > 0) parts.push(`reasoning_chars=${reasoningChars}`); } else { return null; } @@ -1138,14 +1251,34 @@ async function requestSummary(deps: PreflightDeps, system: string, content: stri } if (summary.length < MIN_SUMMARY_CHARS) { const diagnosis = diagnoseEmptySummary(text, json); - deps.log("warn", `[preflight] summary too short (${summary.length} chars): ${diagnosis}`); - return { unusable: diagnosis, transient: !emptySummaryIsSizeDriven(diagnosis) }; + // #2832: a normal-stop empty completion is deterministic — retrying the + // identical payload cannot succeed, so it is neither transient nor + // size-driven; the walk's dead-probe verdict counts it instead. + const deterministic = isDeterministicEmptyCompletion(deps.protocol, json); + deps.log("warn", `[preflight] summary too short (${summary.length} chars): ${diagnosis}${deterministic ? " [deterministic empty completion: normal stop with zero text — retrying the identical payload cannot succeed]" : ""}`); + return deterministic + ? { unusable: diagnosis, transient: false, deterministic: true } + : { unusable: diagnosis, transient: !emptySummaryIsSizeDriven(diagnosis) }; } return { summary }; } const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client disconnected during preflight compression" }; +// #2832: persisted half of the dead-probe verdict (see the constant above). +// Lives in session metadata like #626's learned stream flag so a verdict +// armed on one turn survives into later turns of the same session; the +// (host, model) key makes a stale entry inert after a model or relay switch. +type DeterministicProbeState = { key: string; strikes: number; dead: boolean }; + +function readDeterministicProbeState(metadata: Record): DeterministicProbeState | undefined { + const stored = metadata["deterministicEmptyProbe"]; + if (!stored || typeof stored !== "object") return undefined; + const o = stored as Record; + if (typeof o["key"] !== "string" || typeof o["strikes"] !== "number" || typeof o["dead"] !== "boolean") return undefined; + return { key: o["key"], strikes: o["strikes"], dead: o["dead"] }; +} + // #330: the soft-protected recent zone (preserveRecentMessages / // preserveRecentTokens / most-recent user message) can cover ALL foldable // content when one large recent message pushes the payload over a small @@ -1292,6 +1425,48 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa let summaryCalls = 0; let budgetHit = false; let transientRetryBudget = TRANSIENT_EMPTY_RETRY_BUDGET; + // #2832: dead-probe verdict for the MODEL summary path. Keyed by + // (host, model) — evidence against one relay/model pair says nothing + // about another. Gated off when an external summary lane is configured: + // the walk's probes then run through that executor, not the model path + // this verdict describes. + const probeGated = !deps.externalSummary; + const probeKey = `${safeHost(deps.url) ?? "?"}|${deps.model}`; + let probeState = probeGated ? readDeterministicProbeState(deps.session.metadata) : undefined; + let probeDead = probeGated && probeState?.dead === true && probeState.key === probeKey; + if (probeDead) { + deps.log("info", `[preflight] carrying the dead-probe verdict from a previous turn for ${probeKey}: at most one canary probe this turn; configure compress.externalSummary or switch the model to restore folding (#2832)`); + } + // One canary probe per invocation is always allowed while dead: it is how + // a self-healed relay clears the verdict, and it bounds the waste to one + // billed call per turn instead of today's retry budget per span. + let canarySpent = false; + const stickyFailureDetail = + `the summary probe path is dead for ${probeKey}: ${DETERMINISTIC_EMPTY_STICKY_LIMIT} consecutive completions ended normally (end_turn/stop/completed) with zero answer text — ` + + `the model answered inside its thinking channel or the relay strips text blocks, so every probe bills the span and returns the same empty answer. ` + + `Restore folding by configuring compress.externalSummary (a working summary model), switching the session's model, or fixing the relay's thinking/text handling (#2832)`; + const recordDeterministicEmpty = (): void => { + if (!probeGated) return; + let st = probeState; + if (!st || st.key !== probeKey) st = { key: probeKey, strikes: 0, dead: false }; + st.strikes += 1; + if (!probeDead && st.strikes >= DETERMINISTIC_EMPTY_STICKY_LIMIT) { + probeDead = true; + st.dead = true; + canarySpent = true; + deps.log("warn", `[preflight] dead-probe verdict armed: ${st.strikes} consecutive deterministic empty completions (normal stop, zero text) for ${probeKey}. Every further probe bills the span and returns the same empty answer — stopping further probes this turn (one canary probe per turn until a probe succeeds). Restore folding by configuring compress.externalSummary (a working summary model), switching the session's model, or fixing the relay's thinking/text handling (block fingerprints in the lines above; #2832)`); + } + probeState = st; + deps.session.metadata["deterministicEmptyProbe"] = st; + }; + const clearDeterministicProbe = (): void => { + const st = probeState; + if (!probeGated || !st || st.key !== probeKey) return; + deps.log("info", `[preflight] summary probe for ${probeKey} answered with usable text again — clearing the dead-probe verdict (${st.strikes} strike(s) recorded)`); + delete deps.session.metadata["deterministicEmptyProbe"]; + probeState = undefined; + probeDead = false; + }; let rangesTried = 0; // #2657×#2678 fusion: set when a round's whole external-summary batch died // in the shared pool's queue (deadline, zero dispatched attempts). Later @@ -1753,6 +1928,14 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (budgetHit) break; const skipKey = `${range.startRef}:${range.endRef}`; if (skipSet.has(skipKey)) continue; + // #2832: dead-probe verdict with the canary spent — no range in this + // walk can fold (they all share the same dead model path), so stop + // the walk with the actionable detail instead of probing again. + if (probeGated && probeDead && canarySpent) { + noteSkip(`${skipKey}: summary probe path dead for ${probeKey} — no further probes this turn (#2832)`); + failure = { kind: "exhausted", detail: stickyFailureDetail }; + break; + } const { refToIdx } = refMaps(messages, deps.session.state); if (baselineKnown) { // #1841: mid-walk futility bail. Ranges already consumed, @@ -1816,6 +1999,15 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa break; } if (budgetHit) break; + // #2832: verdict armed mid-walk (this range's own draw was the + // last strike) — abandon the remaining sub-spans CPU-only; they + // would all hit the same wall and the next range's gate stops + // the whole walk. + if (probeGated && probeDead && canarySpent) { + noteSkip(`${skipKey}: summary probe path dead for ${probeKey} — remaining sub-span(s) not probed (#2832)`); + skipSet.add(skipKey); + break; + } const span = spans.pop(); if (!span) break; const [cs, ce] = span; @@ -1945,6 +2137,9 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa break; } summaryCalls += 1; + // #2832: while the verdict is dead this call is the + // invocation's one canary probe. + if (probeGated && probeDead) canarySpent = true; let part = await summarizeRange(deps, chunk, startRef, endRef, perChunkBudget); let transientTries = 0; while ( @@ -1962,11 +2157,13 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa summaryCalls += 1; part = await summarizeRange(deps, chunk, startRef, endRef, perChunkBudget); } + if ("unusable" in part && part.deterministic) recordDeterministicEmpty(); if ("unusable" in part) { outcome = part; break; } if (part.external) externalFold = true; + else clearDeterministicProbe(); parts.push(part.summary); } if (!budgetHit && !outcome && parts.length === chunks.length) { @@ -2070,6 +2267,18 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (summary === null) { const unusableDetail = outcome && "unusable" in outcome ? outcome.unusable : "unknown"; if (outcome) lastUnusableDetail = unusableDetail; + // #2832: a deterministic empty (normal stop, zero text) is not + // size-driven — halving would only re-bill doomed draws + // against the same wall, so skip straight to the give-up. + // The strike may have armed the dead-probe verdict, which + // stops the whole walk at the next range's gate. + const deterministicUnusable = outcome !== undefined && "unusable" in outcome && outcome.deterministic === true; + if (deterministicUnusable) { + deps.log("warn", `[preflight] range ${skipKey} got a deterministic empty summary (${unusableDetail}); skipping without halving — a normal stop with zero text is not size-driven (#2832)`); + noteSkip(`${skipKey}: deterministic empty summary (${unusableDetail})`); + skipSet.add(skipKey); + break; + } const floorUnits = baselineKnown ? 2 * MIN_CHUNK_TOKENS : 2 * minChars; if (ce > cs && spanUnitsOf(messages, cs, ce, countText) >= floorUnits) { deps.log("warn", `[preflight] chunk ${startRef}:${endRef} produced no usable summary (${unusableDetail}); retrying with smaller chunks`); diff --git a/tests/issue2832-preflight-deterministic-empty.test.ts b/tests/issue2832-preflight-deterministic-empty.test.ts new file mode 100644 index 0000000000..fe8ffca237 --- /dev/null +++ b/tests/issue2832-preflight-deterministic-empty.test.ts @@ -0,0 +1,323 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; +process.env.BILI_REPLAY_RETRY_MAX = "1"; +// Zero backoff so any retry legs run instantly. +process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; +process.env.BILI_PREFLIGHT_DEAD_END_COOLDOWN_MS = "400"; + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; +import { diagnoseEmptySummary, isDeterministicEmptyCompletion } from "../src/preflight.ts"; + +// #2832: a relay model (kimi code alias behind an anthropic-shaped gateway) +// answered every preflight summary probe with stop_reason=end_turn and its +// whole answer inside thinking blocks — zero text. Preflight classified that +// as TRANSIENT (no size signal matched), burned two same-span retries plus +// the per-regime retry budget on every span, then let the #726 halving +// cascade re-bill doomed sub-spans: six failed probes over six minutes, ZERO +// folds landed, and the turn fell through to an evidence-only forward that +// billed the full payload. The fix classifies normal-stop-zero-text +// completions as DETERMINISTIC (no same-span retries, no halving), records a +// strike per (host, model) pair in session metadata, arms a dead-probe +// verdict after three consecutive strikes, and lets exactly ONE canary probe +// per invocation detect a self-healed relay (which clears the verdict). + +const SUMMARY_TEXT = + "#2832 TEST SUMMARY: the segment held large log dumps plus several short turns; the raw payload is derivable, so the folded view loses nothing of value."; + +type Call = { summary: boolean; failed: boolean }; + +// The incident shape on the openai wire: HTTP 200 plain JSON, empty content, +// NORMAL stop reason, the answer riding in reasoning_content instead. +function deterministicEmptyJson(res: http.ServerResponse): void { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "chatcmpl-det", object: "chat.completion", model: "muse-spark-1.3-contributor", + choices: [{ index: 0, message: { role: "assistant", content: "", reasoning_content: "R".repeat(2048) }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }, + })); +} + +function okSummaryJson(res: http.ServerResponse): void { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "chatcmpl-ok", object: "chat.completion", model: "muse-spark-1.3-contributor", + choices: [{ index: 0, message: { role: "assistant", content: SUMMARY_TEXT }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + })); +} + +function forwardSse(res: http.ServerResponse): void { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + const line = (o: unknown): void => { res.write(`data: ${JSON.stringify(o)}\n\n`); }; + line({ id: "c_fwd", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }] }); + line({ id: "c_fwd", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 100, completion_tokens: 2 } }); + res.write("data: [DONE]\n\n"); + res.end(); +} + +type ParsedBody = { messages?: unknown; instructions?: unknown }; + +function parseBody(raw: string): ParsedBody { + try { + return JSON.parse(raw) as ParsedBody; + } catch { + return {}; + } +} + +// Summary calls on the openai wire carry exactly [system, user]; forwards +// carry the full conversation. +function isSummaryCall(parsed: ParsedBody): boolean { + return Array.isArray(parsed.messages) && parsed.messages.length === 2 + && (parsed.messages[0] as Record).role === "system"; +} + +function makeUpstream(calls: Call[], failMode: { current: boolean }): http.Server { + return http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const parsed = parseBody(Buffer.concat(chunks).toString("utf8")); + const summary = isSummaryCall(parsed); + const failed = summary && failMode.current; + calls.push({ summary, failed }); + if (failed) deterministicEmptyJson(res); + else if (summary) okSummaryJson(res); + else forwardSse(res); + }); + }); +} + +function startProxy(upstreamPort: number, models: Record): Promise { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + return startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models } }, + modelContextLimit: 10_000, + kernelConfig: defaultConfig(10_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); +} + +async function driveChat(proxyPort: number, upstreamPort: number, session: string, model: string, messages: unknown[]): Promise { + return await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": session }, + body: JSON.stringify({ model, stream: true, messages }), + }); +} + +// Four log-dump blocks separated by short user turns, well over the 10K +// window but far below the #2383 emergency digest bound (19.2x), so the LLM +// fold path runs. Nothing is ever forwarded while the relay is broken, so +// every turn re-enters preflight on the same over-window payload — the +// incident's steady state. +function giantBlocksInput(): unknown[] { + const items: unknown[] = []; + for (let i = 0; i < 4; i++) { + items.push({ role: "user", content: `Question ${i}: what do these logs say?` }); + items.push({ role: "assistant", content: `LOG_BLOCK_${i}_`.repeat(1100) }); + } + items.push({ role: "user", content: "final question here" }); + items.push({ role: "assistant", content: "short tail reply" }); + return items; +} + +type ProbeState = { key: string; strikes: number; dead: boolean }; + +function probeStateOf(sessionId: string): ProbeState | undefined { + const sess = listSessions().find((s) => s.id.includes(sessionId)); + assert.ok(sess, `session ${sessionId} recorded`); + const stored = sess?.metadata?.["deterministicEmptyProbe"]; + if (!stored || typeof stored !== "object") return undefined; + return stored as ProbeState; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +test("#2832 classifier: normal-stop zero-text is deterministic; blips and size signals are not", () => { + const thinkingOnly = { + id: "msg_1", model: "k3-256k", stop_reason: "end_turn", + content: [ + { type: "thinking", thinking: "let me think about this summary..." }, + { type: "redacted_thinking", data: "xxx" }, + ], + }; + assert.equal(isDeterministicEmptyCompletion("anthropic", thinkingOnly), true, "thinking-only end_turn is deterministic"); + assert.equal(isDeterministicEmptyCompletion("anthropic", { ...thinkingOnly, content: [{ type: "text", text: "short" }] }), false, "any answer text is NOT deterministic"); + assert.equal(isDeterministicEmptyCompletion("anthropic", { ...thinkingOnly, stop_reason: "max_tokens" }), false, "max_tokens stays size-driven"); + assert.equal(isDeterministicEmptyCompletion("anthropic", { ...thinkingOnly, content: "not-an-array" }), false, "unrecognized content shape is never deterministic"); + const openaiStopEmpty = { choices: [{ message: { role: "assistant", content: "", reasoning_content: "thought hard" }, finish_reason: "stop" }] }; + assert.equal(isDeterministicEmptyCompletion("openai", openaiStopEmpty), true, "finish_reason=stop with empty content is deterministic"); + assert.equal(isDeterministicEmptyCompletion("openai", { choices: [{ message: { content: "" }, finish_reason: "length" }] }), false, "finish_reason=length stays size-driven"); + assert.equal(isDeterministicEmptyCompletion("openai", { choices: [{ message: { content: "" }, finish_reason: "content_filter" }] }), false, "content_filter stays transient (#1767)"); + assert.equal(isDeterministicEmptyCompletion("openai", { choices: [{ message: { content: "x".repeat(60) }, finish_reason: "stop" }] }), false, "a real (short) answer is not deterministic"); + assert.equal(isDeterministicEmptyCompletion("responses", { status: "completed", output: [] }), true, "completed-with-no-output is deterministic"); + assert.equal(isDeterministicEmptyCompletion("responses", { output: [] }), true, "omitted status stays accepted"); + assert.equal(isDeterministicEmptyCompletion("responses", { status: "incomplete", output: [] }), false, "incomplete stays transient (#780/#784)"); + assert.equal(isDeterministicEmptyCompletion("google", { candidates: [] }), false, "google is out of scope for this classifier"); + assert.equal(isDeterministicEmptyCompletion("anthropic", null), false, "SSE bodies (json=null) are never deterministic here"); + assert.equal(isDeterministicEmptyCompletion("anthropic", "event: x\ndata: {}"), false, "non-object bodies are never deterministic"); +}); + +test("#2832 diagnosis carries the block fingerprint (thinking-only vs stripped-text)", () => { + // The incident shape: anthropic end_turn, answer entirely in thinking. + const anthropicThinkingOnly = JSON.stringify({ + id: "msg_1", model: "k3-256k", stop_reason: "end_turn", + content: [ + { type: "thinking", thinking: "A".repeat(1873) }, + { type: "redacted_thinking", data: "xxx" }, + ], + }); + assert.match( + diagnoseEmptySummary(anthropicThinkingOnly, JSON.parse(anthropicThinkingOnly)), + /plain-JSON completion with empty content \(stop_reason=end_turn, text_blocks=0, thinking_blocks=2, reasoning_chars=1873, answered as model=k3-256k\)/, + ); + // Same stop reason, relay stripped EVERYTHING: no thinking blocks either. + const anthropicStripped = JSON.stringify({ id: "msg_2", model: "k3-256k", stop_reason: "end_turn", content: [] }); + assert.match( + diagnoseEmptySummary(anthropicStripped, JSON.parse(anthropicStripped)), + /plain-JSON completion with empty content \(stop_reason=end_turn, text_blocks=0, answered as model=k3-256k\)/, + ); + const openaiReasoning = JSON.stringify({ + id: "chatcmpl-1", model: "muse-spark", + choices: [{ index: 0, message: { role: "assistant", content: "", reasoning_content: "B".repeat(512) }, finish_reason: "stop" }], + }); + assert.match( + diagnoseEmptySummary(openaiReasoning, JSON.parse(openaiReasoning)), + /plain-JSON completion with empty content \(finish_reason=stop, text_chars=0, reasoning_chars=512, answered as model=muse-spark\)/, + ); + const responsesEmpty = JSON.stringify({ id: "resp_1", status: "completed", output: [], model: "gpt-x" }); + assert.match( + diagnoseEmptySummary(responsesEmpty, JSON.parse(responsesEmpty)), + /plain-JSON completion with empty content \(status=completed, text_parts=0, answered as model=gpt-x\)/, + ); +}); + +// Drives turns until the dead-probe verdict arms (at most one strike per +// probe-able range per turn; the bound below is generous to kernel range +// topology). Returns the turn count. +async function driveUntilArmed( + proxyPort: number, + upstreamPort: number, + session: string, + model: string, + input: unknown[], + maxTurns: number, +): Promise { + for (let turn = 1; turn <= maxTurns; turn++) { + const r = await driveChat(proxyPort, upstreamPort, session, model, input); + const body = await r.text(); + assert.equal(r.status, 502, `turn ${turn} must fail-fast while the relay is broken, got ${r.status}: ${body.slice(0, 300)}`); + if (probeStateOf(session)?.dead === true) return turn; + await sleep(500); + } + throw new Error(`verdict did not arm within ${maxTurns} turns (last state: ${JSON.stringify(probeStateOf(session))})`); +} + +test("#2832 systemic deterministic empties: bounded strikes, sticky verdict, one canary per turn, nothing forwarded", { timeout: 180_000 }, async () => { + const calls: Call[] = []; + const failMode = { current: true }; + const upstream = makeUpstream(calls, failMode); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort, { "muse-spark-1.3-contributor": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + + try { + const input = giantBlocksInput(); + const armedTurn = await driveUntilArmed(proxyPort, upstreamPort, "s2832-dead", "muse-spark-1.3-contributor", input, 6); + assert.ok(armedTurn <= 3, `three strikes must arm the verdict within three turns, took ${armedTurn}`); + + // OLD behavior burned 1 draw + 2 retries per span plus the #726 + // halving cascade for EVERY span of EVERY range EVERY turn (the + // incident: 6+ probes in one minute, unbounded across ranges). NEW + // behavior pays at most one strike per probe-able range per turn. + const summariesArmed = calls.filter((c) => c.summary).length; + assert.ok(summariesArmed >= 3 && summariesArmed <= 6, `total strikes must be bounded (3 to arm, one regime-headroom margin), got ${summariesArmed}`); + assert.ok(!calls.some((c) => !c.summary), "nothing may be forwarded while the relay is broken"); + + const st = probeStateOf("s2832-dead"); + assert.equal(st!.dead, true, "verdict armed"); + assert.ok(st!.key.includes("muse-spark-1.3-contributor"), "verdict key carries the model"); + + // Past the dead-end cooldown: the NEXT turn spends EXACTLY ONE canary + // probe (old behavior re-burned the full retry budget per span). + await sleep(500); + const r2 = await driveChat(proxyPort, upstreamPort, "s2832-dead", "muse-spark-1.3-contributor", input); + assert.equal(r2.status, 502, `carried verdict must still fail-fast, got ${r2.status}`); + const j2 = (await r2.json()) as { error?: { code?: string; message?: string } }; + assert.equal(j2.error?.code, "preflight_compress_failed"); + const summariesAfterCanary = calls.filter((c) => c.summary).length; + assert.equal(summariesAfterCanary, summariesArmed + 1, `a carried verdict allows exactly ONE canary probe per turn, got +${summariesAfterCanary - summariesArmed}`); + assert.ok(!calls.slice(summariesArmed).some((c) => !c.summary), "still nothing forwarded"); + assert.equal(probeStateOf("s2832-dead")?.dead, true, "a failed canary keeps the verdict armed"); + } finally { + proxy.close(); + upstream.close(); + await new Promise((resolve, reject) => { + void Promise.allSettled([once(proxy, "close"), once(upstream, "close")]).then(() => resolve(), reject); + }); + } +}); + +test("#2832 self-healed relay: the canary clears the verdict and folding resumes", { timeout: 180_000 }, async () => { + const calls: Call[] = []; + const failMode = { current: true }; + const upstream = makeUpstream(calls, failMode); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort, { "muse-spark-1.3-contributor": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + + try { + const input = giantBlocksInput(); + await driveUntilArmed(proxyPort, upstreamPort, "s2832-heal", "muse-spark-1.3-contributor", input, 6); + + // The relay self-heals between turns. + failMode.current = false; + await sleep(500); + const r3 = await driveChat(proxyPort, upstreamPort, "s2832-heal", "muse-spark-1.3-contributor", input); + assert.equal(r3.status, 200, `the canary must detect the healed relay and resume folding, got ${r3.status}: ${(await r3.text()).slice(0, 300)}`); + assert.equal(probeStateOf("s2832-heal"), undefined, "a successful probe must clear the verdict"); + assert.ok(calls.some((c) => !c.summary), "the folded payload was forwarded"); + const okSummaries = calls.filter((c) => c.summary && !c.failed).length; + assert.ok(okSummaries >= 1, "at least the canary summary succeeded"); + } finally { + proxy.close(); + upstream.close(); + await new Promise((resolve, reject) => { + void Promise.allSettled([once(proxy, "close"), once(upstream, "close")]).then(() => resolve(), reject); + }); + } +}); diff --git a/tests/preflight-empty-summary.test.ts b/tests/preflight-empty-summary.test.ts index 19db72370c..5cd1f34c28 100644 --- a/tests/preflight-empty-summary.test.ts +++ b/tests/preflight-empty-summary.test.ts @@ -418,14 +418,17 @@ test("#987 diagnoseEmptySummary names plain-JSON empty completions precisely", ( id: "chatcmpl-1789806312367", object: "chat.completion", model: "muse-spark-1.3-contributor", choices: [{ index: 0, message: { role: "assistant", content: "" }, finish_reason: "stop" }], }); + // #2832: the diagnosis now carries the block-level fingerprint — the + // text_chars/text_blocks parts are what separate a thinking-only answer + // from a relay that stripped the text. assert.match( diagnoseEmptySummary(openai, JSON.parse(openai)), - /plain-JSON completion with empty content \(finish_reason=stop, answered as model=muse-spark-1\.3-contributor\)/, + /plain-JSON completion with empty content \(finish_reason=stop, text_chars=0, answered as model=muse-spark-1\.3-contributor\)/, ); const anthropic = JSON.stringify({ id: "msg_1", model: "claude-x", content: [], stop_reason: "max_tokens" }); assert.match( diagnoseEmptySummary(anthropic, JSON.parse(anthropic)), - /plain-JSON completion with empty content \(stop_reason=max_tokens, answered as model=claude-x\)/, + /plain-JSON completion with empty content \(stop_reason=max_tokens, text_blocks=0, answered as model=claude-x\)/, ); // #2309: an explicit Responses terminal state now gets its own precise // diagnosis (the old string claimed "empty content", which is false when diff --git a/tests/preflight-multi-range.test.ts b/tests/preflight-multi-range.test.ts index 9a530e51a0..138b1f6e42 100644 --- a/tests/preflight-multi-range.test.ts +++ b/tests/preflight-multi-range.test.ts @@ -57,7 +57,13 @@ type Call = { stream: boolean; body: string }; // Mock Anthropic upstream. Main requests are stream:true (answered with SSE); // preflight summarization calls are stream:false (answered with a JSON message // whose text is SUMMARY_TEXT when usable(idx) else empty — the proxy treats an -// empty/too-short summary as unusable and skips that range). +// empty/too-short summary as unusable and skips that range). The unusable +// shape is stop_reason=max_tokens ON PURPOSE: #2832 reclassified normal-stop +// zero-text completions (end_turn + empty) into the DETERMINISTIC class whose +// dead-probe verdict stops the walk after three strikes — this file must keep +// pinning the walk/budget mechanics themselves, so its failures stay on the +// size-driven halving path the "2 calls per failed range" arithmetic below +// describes. function makeUpstream(usable: (idx: number) => boolean): { server: http.Server; calls: Call[] } { const calls: Call[] = []; let summaryIdx = 0; @@ -78,7 +84,7 @@ function makeUpstream(usable: (idx: number) => boolean): { server: http.Server; res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ id: "msg_summary", type: "message", role: "assistant", model: "claude-small", - content: [{ type: "text", text }], stop_reason: "end_turn", + content: [{ type: "text", text }], stop_reason: "max_tokens", usage: { input_tokens: 500, output_tokens: 50 }, })); } diff --git a/tests/preflight-transient-empty-retry.test.ts b/tests/preflight-transient-empty-retry.test.ts index 0d19ba467b..88f570be17 100644 --- a/tests/preflight-transient-empty-retry.test.ts +++ b/tests/preflight-transient-empty-retry.test.ts @@ -256,7 +256,9 @@ test("#1767 empty-summary classifier: transient vs size-driven", () => { choices: [{ index: 0, message: { role: "assistant", content: "" }, finish_reason: "content_filter" }], }); const cfDetail = diagnoseEmptySummary(cf, JSON.parse(cf)); - assert.match(cfDetail, /plain-JSON completion with empty content \(finish_reason=content_filter, answered as model=deepseek\/deepseek-v4\.1-flash\)/); + // #2832: the fingerprint gained a text_chars part; content_filter stays + // TRANSIENT (it is not a normal stop) despite the zero-text body. + assert.match(cfDetail, /plain-JSON completion with empty content \(finish_reason=content_filter, text_chars=0, answered as model=deepseek\/deepseek-v4\.1-flash\)/); assert.equal(emptySummaryIsSizeDriven(cfDetail), false, "content_filter empty completion is transient"); const len = JSON.stringify({ choices: [{ index: 0, message: { role: "assistant", content: "" }, finish_reason: "length" }] }); assert.equal(emptySummaryIsSizeDriven(diagnoseEmptySummary(len, JSON.parse(len))), true, "finish_reason=length is size-driven");