diff --git a/src/loop/core.ts b/src/loop/core.ts index 079b5b3a4f..09866d16d9 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -1126,9 +1126,12 @@ export async function* runCompressLoop( finishReason !== "failed" && finishReason !== "incomplete" && finishReason !== "error" && - assistantText.length === 0 && + // #732 follow-up: trim, so whitespace-only text ("\n\n") must not bar + // the retry — it is invisible to host turn semantics, same class as + // the reasoning prefix. (The truncation gates keep the raw + // forwardedVisible: they need forwarded-not-parsed semantics.) + assistantText.trim().length === 0 && calls.length === 0 && - !forwardedVisible && !signal?.aborted ) { degenerateRetried = true; @@ -1227,6 +1230,28 @@ export async function* runCompressLoop( break; } + // #732 follow-up (observability only): the one-shot PER-REQUEST retry budget + // (degenerateRetried — shared with the #2303 draft-tail path, never reset per + // round) was already spent earlier in this request, and THIS round still ended + // degenerate — zero visible output, no tool call — so nothing will retry it. + // The host nudge is the only remaining safety net; make the fleet able to + // count these pass-throughs. + if ( + degenerateRetried && + sawDone && + !truncatedDone && + !suppressCompletion && + typeof finishReason === "string" && + finishReason !== "failed" && + finishReason !== "incomplete" && + finishReason !== "error" && + assistantText.trim().length === 0 && + calls.length === 0 && + !signal?.aborted + ) { + loggerLog("warn", `[acp-loop] degenerate-turn DOUBLE-degenerate: degenerate terminal turn after the per-request retry budget was already spent, round ${round} (session ${ctx.session.id}) — passing through; host nudge is the remaining safety net (#732 follow-up)`); + } + if ( usage.inputTokens !== undefined || usage.outputTokens !== undefined || diff --git a/src/stream.ts b/src/stream.ts index 27123b2c8f..a4c8c32840 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -280,6 +280,13 @@ function armedRefusalReceipt(n: number, decayMinutes: number): string { return `[Compression refused: this call carried no usable ranges and the compress circuit breaker is still armed (${n} consecutive failures) — while it stays armed, repeat parse-failure lectures are withheld. Format reminder: 'content' must be a non-empty array whose elements are {startId, endId, summary} objects with mNNNNN refs from your visible history. The breaker disarms on one successful fold or ${decayMinutes} minutes without a failing call. Continue your task without compressing.]`; } +// #5986 review fix: the observed failure mode after a compress rejection is the +// model getting entangled in compression bookkeeping and then stopping out — +// every parse-failure receipt ends with this explicit return-to-task instruction +// (the armed-breaker one-liner above already carries its own "Continue your task +// without compressing.", so it is left untouched). +const RETURN_TO_TASK_SUFFIX = " This failure only concerns the compress call. Return to the original task now and continue it (visible output or a tool call)."; + /** #2146: record one total compress failure (any spec) and return "" while * healthy, or the pause paragraph to embed in the receipt once the * consecutive-failure threshold is reached. Only failed-call receipts carry @@ -1103,11 +1110,11 @@ export function applyRanges(parsed: ReturnType, ctx: return compressResult(armedRefusalReceipt(armedDetail.n, armedDetail.decayMinutes), "refused", 0, `parse:${diagnostics.kind}`); } if (isEmptyCall) { - return compressResult(`[Compression FAILED: the call carried no content at all (kind=${diagnostics.kind}) — an empty compress() compresses nothing and can never succeed. Do NOT re-issue an empty call; if you meant to compress, put content as ONE non-empty string in that SAME single call — each block headed by a 'mNNNNN–mNNNNN optional topic' line followed by its summary markdown (a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string).${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`); + return compressResult(`[Compression FAILED: the call carried no content at all (kind=${diagnostics.kind}) — an empty compress() compresses nothing and can never succeed. Do NOT re-issue an empty call; if you meant to compress, put content as ONE non-empty string in that SAME single call — each block headed by a 'mNNNNN–mNNNNN optional topic' line followed by its summary markdown (a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string).${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`); } if (argCorruption) { const truncNote = diagnostics.kind === "truncated" ? " (looks truncated)" : ""; - return compressResult(`[Compression FAILED: the call's arguments (${argLen} chars) were not parseable JSON${truncNote} — the intended content was lost and nothing was compressed. Re-issue ONE compress call whose arguments parse as a single object with a STRING 'content' field: {"content": ""}. Never send 'content' as an actual JSON array or nested object — it is one plain string.${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`); + return compressResult(`[Compression FAILED: the call's arguments (${argLen} chars) were not parseable JSON${truncNote} — the intended content was lost and nothing was compressed. Re-issue ONE compress call whose arguments parse as a single object with a STRING 'content' field: {"content": ""}. Never send 'content' as an actual JSON array or nested object — it is one plain string.${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`); } const recovery = loopGuard === "" ? compressionRecovery(ctx) : { hint: "", canRetry: false }; const retry = recovery.canRetry ? " Re-issue the compress call using those eligible refs." : ""; @@ -1119,7 +1126,7 @@ export function applyRanges(parsed: ReturnType, ctx: const notArrayHint = diagnostics.kind === "content-not-array" ? " If your 'content' string encoded a single JSON object, wrap it in square brackets so it encodes an ARRAY of entries (\"[...]\"), or use the line-form headers below." : ""; - return compressResult(`[Compression FAILED: no valid ranges parsed (kind=${diagnostics.kind}, dropped=${diagnostics.invalidItems}).${why}${notArrayHint} compress takes ONE 'content' STRING holding every range: each block starts with its header line 'mNNNNN–mNNNNN optional topic' followed by that block's summary markdown; a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string. startId/endId are mNNNNN message refs from the conversation (call acp_status to see current refs).${recovery.hint}${retry}${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`); + return compressResult(`[Compression FAILED: no valid ranges parsed (kind=${diagnostics.kind}, dropped=${diagnostics.invalidItems}).${why}${notArrayHint} compress takes ONE 'content' STRING holding every range: each block starts with its header line 'mNNNNN–mNNNNN optional topic' followed by that block's summary markdown; a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string. startId/endId are mNNNNN message refs from the conversation (call acp_status to see current refs).${recovery.hint}${retry}${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`); } // #847: detect reversed refs as SUBMITTED, before #1001 normalization // rewrites them (order matters — normalizeRangeOrder mutates in place). diff --git a/tests/issue411-abort-usage.test.ts b/tests/issue411-abort-usage.test.ts index f2b50fd24b..ef95cc79b8 100644 --- a/tests/issue411-abort-usage.test.ts +++ b/tests/issue411-abort-usage.test.ts @@ -133,10 +133,15 @@ function modelBody(stream: boolean): string { return JSON.stringify({ model: "i411-model", max_tokens: 8192, stream, messages: [{ role: "user", content: "hello" }] }); } -async function waitFor(cond: () => boolean, ms = 3000): Promise { +// #4161: the deadline message follows the repo-wide 'timeout waiting for …' +// idiom so a load-starved windows-latest shard classifies as a recoverable +// timing-window failure (one bounded retry) instead of a hard concrete fail — +// this file's old "condition not met within timeout" literal was the only one +// in the suite outside that classification. +async function waitFor(cond: () => boolean, what: string, ms = 3000): Promise { const t0 = Date.now(); while (!cond()) { - if (Date.now() - t0 > ms) throw new Error("condition not met within timeout"); + if (Date.now() - t0 > ms) throw new Error(`timeout waiting for ${what}`); await new Promise((r) => setTimeout(r, 25)); } } @@ -154,12 +159,16 @@ test("#411 plugin stream: client abort keeps sniffed usage, logs info (not error await reader.read(); await reader.read(); ac.abort(); - await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234)); - await waitFor(() => _liveUpstreamTimersForTest() === 0); + await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234), "sniffed usage (lastInputTokens=1234) to land"); + await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear"); // #2305 review: the info abort log is a SEPARATE async side effect // that can land after the timer clears — waiting avoids the // synchronous-assert flake seen on a loaded windows-latest runner. - await waitFor(() => captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream"))); + // #4161 follow-up: that async delivery still exceeded 3s on a loaded + // windows-latest/22 shard (PR #5986 run, 2026-11-09) — widen THIS + // budget to 8s; the message idiom above earns the bounded retry as + // belt and suspenders. + await waitFor(() => captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream")), "client aborted mid-stream info log", 8000); assert.ok(captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream")), `expected info abort log, got: ${JSON.stringify(captured)}`); assert.ok(!captured.some((l) => l.level === "error" && l.msg.includes("AbortError")), `no bare AbortError error log, got: ${JSON.stringify(captured.filter((l) => l.level === "error"))}`); } finally { @@ -179,7 +188,7 @@ test("#411 abort storm: repeated client cancels do not accumulate upstream timer await reader.read(); await reader.read(); ac.abort(); - await waitFor(() => _liveUpstreamTimersForTest() === 0); + await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timers to clear after cancel round"); } assert.equal(_liveUpstreamTimersForTest(), 0); } finally { @@ -204,8 +213,8 @@ test("#411 upstream cut with client alive: usage still lands, failure is a real } catch { // upstream socket destroyed mid-stream — the client sees a cut } - await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234)); - await waitFor(() => _liveUpstreamTimersForTest() === 0); + await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234), "sniffed usage (lastInputTokens=1234) to land"); + await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear"); assert.ok(!captured.some((l) => l.msg.includes("client aborted mid-stream")), "client-alive cut must not be classified as a client abort"); } finally { setLogCapture(null); @@ -229,7 +238,7 @@ test("#411 plugin non-stream: client abort mid-body does not crash and clears th // aborted mid-body — expected } }); - await waitFor(() => _liveUpstreamTimersForTest() === 0); + await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear"); } finally { await rig.closeAll(); } @@ -246,7 +255,7 @@ test("#411 anonymous passthrough: client abort clears the timer and logs info", const reader = res.body!.getReader(); await reader.read(); ac.abort(); - await waitFor(() => _liveUpstreamTimersForTest() === 0); + await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear"); assert.ok(!captured.some((l) => l.level === "error" && l.msg.includes("AbortError")), `no bare AbortError error log, got: ${JSON.stringify(captured.filter((l) => l.level === "error"))}`); } finally { setLogCapture(null); diff --git a/tests/loop-degenerate-retry.test.ts b/tests/loop-degenerate-retry.test.ts index 5608035973..5f340a634d 100644 --- a/tests/loop-degenerate-retry.test.ts +++ b/tests/loop-degenerate-retry.test.ts @@ -6,6 +6,7 @@ import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter } from "../src/loop/index.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; import { getCacheLedger } from "../src/cache-ledger.ts"; +import { setLogCapture } from "../src/logger.ts"; // #732: invisible single-retry for a degenerate terminal turn (reasoning-only // completion, zero visible text, zero tool calls) that reached the client with @@ -191,6 +192,88 @@ test("#821 O2: genuinely empty (no-reasoning) terminal turn is NOT retried", asy assert.equal(fetchCalls, 0, "sawThinking=false → the empty turn passes through untouched"); }); +// #732 follow-up: a thinking model ends the turn with reasoning + +// whitespace-only text ("\n\n") and no tool call. The 2-char text made the +// #732 gate read "visible output reached" (assistantText.length>0 && +// forwardedVisible) and the retry never fired — the host session stalled +// until its daemon nudge, which the model also answered with a bare "\n\n". + +const OPENAI_THINKING_WSPRIBE = openaiSse([ + { id: "chatcmpl_w1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }] }, + { id: "chatcmpl_w1", choices: [{ index: 0, delta: { reasoning_content: "let me check the logs first to understand what happened" }, finish_reason: null }] }, + { id: "chatcmpl_w1", choices: [{ index: 0, delta: { content: "\n\n" }, finish_reason: null }] }, + { id: "chatcmpl_w1", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, +]); + +const OPENAI_THINKING_REALTEXT = openaiSse([ + { id: "chatcmpl_w2", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }] }, + { id: "chatcmpl_w2", choices: [{ index: 0, delta: { reasoning_content: "the user asked a question" }, finish_reason: null }] }, + { id: "chatcmpl_w2", choices: [{ index: 0, delta: { content: "Here is the answer you asked for." }, finish_reason: null }] }, + { id: "chatcmpl_w2", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, +]); + +test("#732-follow-up W1: reasoning + whitespace-only text + stop → the #732 retry now fires (was bypassed via length 2)", async () => { + const { out, fetchCalls, bodies } = await drainOpenai(OPENAI_THINKING_WSPRIBE, [OPENAI_GOOD], "deg-w1"); + assert.equal(fetchCalls, 1, "exactly one degenerate auto-retry fired despite the forwarded \"\\n\\n\""); + assert.ok( + bodies[0].includes("ended without a tool call"), + "the retry body carries the ephemeral continuation nudge", + ); + assert.ok(out.includes("let me check the logs"), "the thinking prefix was streamed to the client"); + assert.ok(out.includes("\n\n"), "the whitespace-only text reached the client"); + assert.ok(out.includes("continued after nudge"), "the retried turn's content was delivered on the same stream"); +}); + +test("#732-follow-up W2: reasoning + real visible text + stop is NOT retried (a delivered answer is not degenerate)", async () => { + const { out, fetchCalls } = await drainOpenai(OPENAI_THINKING_REALTEXT, [OPENAI_GOOD], "deg-w2"); + assert.equal(fetchCalls, 0, "non-whitespace visible text bars the retry — no false positive on a normal answer"); + assert.ok(out.includes("Here is the answer"), "the normal answer passes through untouched"); +}); + +// #732 follow-up review: the double-degenerate pass-through (per-request budget +// already spent, round still ended degenerate) is observable only through its +// warn line — pin both directions so a regression that drops or spams it fails +// here instead of silently vanishing from fleet logs. + +test("#732-follow-up R1: double-degenerate pass-through emits exactly one DOUBLE-degenerate warn", async () => { + const lines: Array<{ level: string; msg: string }> = []; + setLogCapture((level, msg) => { lines.push({ level, msg }); }); + try { + // Re-request degenerate → retry fires (spends the per-request budget); + // the retry comes back degenerate too → passes through with the warn. + const { fetchCalls } = await drain([ROUND_DEGENERATE, ROUND_DEGENERATE], "deg-r1"); + assert.equal(fetchCalls, 2, "re-request + one retry"); + } finally { + setLogCapture(null); + } + const warns = lines.filter((l) => l.level === "warn" && l.msg.includes("DOUBLE-degenerate")); + assert.equal(warns.length, 1, "exactly one double-degenerate warn"); + assert.ok( + warns[0].msg.includes("retry budget was already spent"), + "the warn attributes the pass-through to the spent per-request budget", + ); +}); + +test("#732-follow-up R2: a SUCCESSFUL retry emits no DOUBLE-degenerate warn", async () => { + const lines: Array<{ level: string; msg: string }> = []; + setLogCapture((level, msg) => { lines.push({ level, msg }); }); + try { + const { fetchCalls } = await drain([ROUND_DEGENERATE, ROUND_GOOD], "deg-r2"); + assert.equal(fetchCalls, 2, "re-request + one successful retry"); + } finally { + setLogCapture(null); + } + assert.deepEqual( + lines.filter((l) => l.msg.includes("DOUBLE-degenerate")), + [], + "no double-degenerate warn when the retry recovered", + ); + assert.ok( + lines.some((l) => l.level === "warn" && l.msg.includes("degenerate-turn auto-retry")), + "the auto-retry warn fired — the capture is live, so the negative above is meaningful", + ); +}); + // #1862: the degenerate-retry boundary on the Responses wire misfired in two ways. // A: a completed reasoning+custom_tool_call turn carries executable output, but the // adapter counted zero tool calls, so the gate treated it as degenerate and re-issued