Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions src/loop/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1126,9 +1126,12 @@ export async function* runCompressLoop(
finishReason !== "failed" &&
finishReason !== "incomplete" &&
finishReason !== "error" &&
assistantText.length === 0 &&
// #732 follow-up: trim, so whitespace-only text ("\n\n") must not bar
// the retry — it is invisible to host turn semantics, same class as
// the reasoning prefix. (The truncation gates keep the raw
// forwardedVisible: they need forwarded-not-parsed semantics.)
assistantText.trim().length === 0 &&
calls.length === 0 &&
!forwardedVisible &&
!signal?.aborted
) {
degenerateRetried = true;
Expand Down Expand Up @@ -1227,6 +1230,28 @@ export async function* runCompressLoop(
break;
}

// #732 follow-up (observability only): the one-shot PER-REQUEST retry budget
// (degenerateRetried — shared with the #2303 draft-tail path, never reset per
// round) was already spent earlier in this request, and THIS round still ended
// degenerate — zero visible output, no tool call — so nothing will retry it.
// The host nudge is the only remaining safety net; make the fleet able to
// count these pass-throughs.
if (
degenerateRetried &&
sawDone &&
!truncatedDone &&
!suppressCompletion &&
typeof finishReason === "string" &&
finishReason !== "failed" &&
finishReason !== "incomplete" &&
finishReason !== "error" &&
assistantText.trim().length === 0 &&
calls.length === 0 &&
!signal?.aborted
) {
loggerLog("warn", `[acp-loop] degenerate-turn DOUBLE-degenerate: degenerate terminal turn after the per-request retry budget was already spent, round ${round} (session ${ctx.session.id}) — passing through; host nudge is the remaining safety net (#732 follow-up)`);
}

if (
usage.inputTokens !== undefined ||
usage.outputTokens !== undefined ||
Expand Down
13 changes: 10 additions & 3 deletions src/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,13 @@ function armedRefusalReceipt(n: number, decayMinutes: number): string {
return `[Compression refused: this call carried no usable ranges and the compress circuit breaker is still armed (${n} consecutive failures) — while it stays armed, repeat parse-failure lectures are withheld. Format reminder: 'content' must be a non-empty array whose elements are {startId, endId, summary} objects with mNNNNN refs from your visible history. The breaker disarms on one successful fold or ${decayMinutes} minutes without a failing call. Continue your task without compressing.]`;
}

// #5986 review fix: the observed failure mode after a compress rejection is the
// model getting entangled in compression bookkeeping and then stopping out —
// every parse-failure receipt ends with this explicit return-to-task instruction
// (the armed-breaker one-liner above already carries its own "Continue your task
// without compressing.", so it is left untouched).
const RETURN_TO_TASK_SUFFIX = " This failure only concerns the compress call. Return to the original task now and continue it (visible output or a tool call).";

/** #2146: record one total compress failure (any spec) and return "" while
* healthy, or the pause paragraph to embed in the receipt once the
* consecutive-failure threshold is reached. Only failed-call receipts carry
Expand Down Expand Up @@ -1103,11 +1110,11 @@ export function applyRanges(parsed: ReturnType<typeof parseCompressInput>, ctx:
return compressResult(armedRefusalReceipt(armedDetail.n, armedDetail.decayMinutes), "refused", 0, `parse:${diagnostics.kind}`);
}
if (isEmptyCall) {
return compressResult(`[Compression FAILED: the call carried no content at all (kind=${diagnostics.kind}) — an empty compress() compresses nothing and can never succeed. Do NOT re-issue an empty call; if you meant to compress, put content as ONE non-empty string in that SAME single call — each block headed by a 'mNNNNN–mNNNNN optional topic' line followed by its summary markdown (a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string).${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`);
return compressResult(`[Compression FAILED: the call carried no content at all (kind=${diagnostics.kind}) — an empty compress() compresses nothing and can never succeed. Do NOT re-issue an empty call; if you meant to compress, put content as ONE non-empty string in that SAME single call — each block headed by a 'mNNNNN–mNNNNN optional topic' line followed by its summary markdown (a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string).${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`);
}
if (argCorruption) {
const truncNote = diagnostics.kind === "truncated" ? " (looks truncated)" : "";
return compressResult(`[Compression FAILED: the call's arguments (${argLen} chars) were not parseable JSON${truncNote} — the intended content was lost and nothing was compressed. Re-issue ONE compress call whose arguments parse as a single object with a STRING 'content' field: {"content": "<each block headed 'mNNNNN–mNNNNN optional topic', its summary on the following lines>"}. Never send 'content' as an actual JSON array or nested object — it is one plain string.${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`);
return compressResult(`[Compression FAILED: the call's arguments (${argLen} chars) were not parseable JSON${truncNote} — the intended content was lost and nothing was compressed. Re-issue ONE compress call whose arguments parse as a single object with a STRING 'content' field: {"content": "<each block headed 'mNNNNN–mNNNNN optional topic', its summary on the following lines>"}. Never send 'content' as an actual JSON array or nested object — it is one plain string.${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`);
}
const recovery = loopGuard === "" ? compressionRecovery(ctx) : { hint: "", canRetry: false };
const retry = recovery.canRetry ? " Re-issue the compress call using those eligible refs." : "";
Expand All @@ -1119,7 +1126,7 @@ export function applyRanges(parsed: ReturnType<typeof parseCompressInput>, ctx:
const notArrayHint = diagnostics.kind === "content-not-array"
? " If your 'content' string encoded a single JSON object, wrap it in square brackets so it encodes an ARRAY of entries (\"[...]\"), or use the line-form headers below."
: "";
return compressResult(`[Compression FAILED: no valid ranges parsed (kind=${diagnostics.kind}, dropped=${diagnostics.invalidItems}).${why}${notArrayHint} compress takes ONE 'content' STRING holding every range: each block starts with its header line 'mNNNNN–mNNNNN optional topic' followed by that block's summary markdown; a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string. startId/endId are mNNNNN message refs from the conversation (call acp_status to see current refs).${recovery.hint}${retry}${guard}${loopGuard}]`, "refused", 0, `parse:${diagnostics.kind}`);
return compressResult(`[Compression FAILED: no valid ranges parsed (kind=${diagnostics.kind}, dropped=${diagnostics.invalidItems}).${why}${notArrayHint} compress takes ONE 'content' STRING holding every range: each block starts with its header line 'mNNNNN–mNNNNN optional topic' followed by that block's summary markdown; a JSON-encoded array of {startId, endId, summary} objects is also accepted INSIDE that same string. startId/endId are mNNNNN message refs from the conversation (call acp_status to see current refs).${recovery.hint}${retry}${guard}${loopGuard}${RETURN_TO_TASK_SUFFIX}]`, "refused", 0, `parse:${diagnostics.kind}`);
}
// #847: detect reversed refs as SUBMITTED, before #1001 normalization
// rewrites them (order matters — normalizeRangeOrder mutates in place).
Expand Down
29 changes: 19 additions & 10 deletions tests/issue411-abort-usage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,15 @@ function modelBody(stream: boolean): string {
return JSON.stringify({ model: "i411-model", max_tokens: 8192, stream, messages: [{ role: "user", content: "hello" }] });
}

async function waitFor(cond: () => boolean, ms = 3000): Promise<void> {
// #4161: the deadline message follows the repo-wide 'timeout waiting for …'
// idiom so a load-starved windows-latest shard classifies as a recoverable
// timing-window failure (one bounded retry) instead of a hard concrete fail —
// this file's old "condition not met within timeout" literal was the only one
// in the suite outside that classification.
async function waitFor(cond: () => boolean, what: string, ms = 3000): Promise<void> {
const t0 = Date.now();
while (!cond()) {
if (Date.now() - t0 > ms) throw new Error("condition not met within timeout");
if (Date.now() - t0 > ms) throw new Error(`timeout waiting for ${what}`);
await new Promise((r) => setTimeout(r, 25));
}
}
Expand All @@ -154,12 +159,16 @@ test("#411 plugin stream: client abort keeps sniffed usage, logs info (not error
await reader.read();
await reader.read();
ac.abort();
await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234));
await waitFor(() => _liveUpstreamTimersForTest() === 0);
await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234), "sniffed usage (lastInputTokens=1234) to land");
await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear");
// #2305 review: the info abort log is a SEPARATE async side effect
// that can land after the timer clears — waiting avoids the
// synchronous-assert flake seen on a loaded windows-latest runner.
await waitFor(() => captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream")));
// #4161 follow-up: that async delivery still exceeded 3s on a loaded
// windows-latest/22 shard (PR #5986 run, 2026-11-09) — widen THIS
// budget to 8s; the message idiom above earns the bounded retry as
// belt and suspenders.
await waitFor(() => captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream")), "client aborted mid-stream info log", 8000);
assert.ok(captured.some((l) => l.level === "info" && l.msg.includes("client aborted mid-stream")), `expected info abort log, got: ${JSON.stringify(captured)}`);
assert.ok(!captured.some((l) => l.level === "error" && l.msg.includes("AbortError")), `no bare AbortError error log, got: ${JSON.stringify(captured.filter((l) => l.level === "error"))}`);
} finally {
Expand All @@ -179,7 +188,7 @@ test("#411 abort storm: repeated client cancels do not accumulate upstream timer
await reader.read();
await reader.read();
ac.abort();
await waitFor(() => _liveUpstreamTimersForTest() === 0);
await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timers to clear after cancel round");
}
assert.equal(_liveUpstreamTimersForTest(), 0);
} finally {
Expand All @@ -204,8 +213,8 @@ test("#411 upstream cut with client alive: usage still lands, failure is a real
} catch {
// upstream socket destroyed mid-stream — the client sees a cut
}
await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234));
await waitFor(() => _liveUpstreamTimersForTest() === 0);
await waitFor(() => listSessions().some((s) => s.stats.lastInputTokens === 1234), "sniffed usage (lastInputTokens=1234) to land");
await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear");
assert.ok(!captured.some((l) => l.msg.includes("client aborted mid-stream")), "client-alive cut must not be classified as a client abort");
} finally {
setLogCapture(null);
Expand All @@ -229,7 +238,7 @@ test("#411 plugin non-stream: client abort mid-body does not crash and clears th
// aborted mid-body — expected
}
});
await waitFor(() => _liveUpstreamTimersForTest() === 0);
await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear");
} finally {
await rig.closeAll();
}
Expand All @@ -246,7 +255,7 @@ test("#411 anonymous passthrough: client abort clears the timer and logs info",
const reader = res.body!.getReader();
await reader.read();
ac.abort();
await waitFor(() => _liveUpstreamTimersForTest() === 0);
await waitFor(() => _liveUpstreamTimersForTest() === 0, "upstream timer to clear");
assert.ok(!captured.some((l) => l.level === "error" && l.msg.includes("AbortError")), `no bare AbortError error log, got: ${JSON.stringify(captured.filter((l) => l.level === "error"))}`);
} finally {
setLogCapture(null);
Expand Down
83 changes: 83 additions & 0 deletions tests/loop-degenerate-retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { Session } from "../src/session.ts";
import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter } from "../src/loop/index.ts";
import { buildCompressSystemPrompt } from "../src/compress-tool.ts";
import { getCacheLedger } from "../src/cache-ledger.ts";
import { setLogCapture } from "../src/logger.ts";

// #732: invisible single-retry for a degenerate terminal turn (reasoning-only
// completion, zero visible text, zero tool calls) that reached the client with
Expand Down Expand Up @@ -191,6 +192,88 @@ test("#821 O2: genuinely empty (no-reasoning) terminal turn is NOT retried", asy
assert.equal(fetchCalls, 0, "sawThinking=false → the empty turn passes through untouched");
});

// #732 follow-up: a thinking model ends the turn with reasoning +
// whitespace-only text ("\n\n") and no tool call. The 2-char text made the
// #732 gate read "visible output reached" (assistantText.length>0 &&
// forwardedVisible) and the retry never fired — the host session stalled
// until its daemon nudge, which the model also answered with a bare "\n\n".

const OPENAI_THINKING_WSPRIBE = openaiSse([
{ id: "chatcmpl_w1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }] },
{ id: "chatcmpl_w1", choices: [{ index: 0, delta: { reasoning_content: "let me check the logs first to understand what happened" }, finish_reason: null }] },
{ id: "chatcmpl_w1", choices: [{ index: 0, delta: { content: "\n\n" }, finish_reason: null }] },
{ id: "chatcmpl_w1", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] },
]);

const OPENAI_THINKING_REALTEXT = openaiSse([
{ id: "chatcmpl_w2", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }] },
{ id: "chatcmpl_w2", choices: [{ index: 0, delta: { reasoning_content: "the user asked a question" }, finish_reason: null }] },
{ id: "chatcmpl_w2", choices: [{ index: 0, delta: { content: "Here is the answer you asked for." }, finish_reason: null }] },
{ id: "chatcmpl_w2", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] },
]);

test("#732-follow-up W1: reasoning + whitespace-only text + stop → the #732 retry now fires (was bypassed via length 2)", async () => {
const { out, fetchCalls, bodies } = await drainOpenai(OPENAI_THINKING_WSPRIBE, [OPENAI_GOOD], "deg-w1");
assert.equal(fetchCalls, 1, "exactly one degenerate auto-retry fired despite the forwarded \"\\n\\n\"");
assert.ok(
bodies[0].includes("ended without a tool call"),
"the retry body carries the ephemeral continuation nudge",
);
assert.ok(out.includes("let me check the logs"), "the thinking prefix was streamed to the client");
assert.ok(out.includes("\n\n"), "the whitespace-only text reached the client");
assert.ok(out.includes("continued after nudge"), "the retried turn's content was delivered on the same stream");
});

test("#732-follow-up W2: reasoning + real visible text + stop is NOT retried (a delivered answer is not degenerate)", async () => {
const { out, fetchCalls } = await drainOpenai(OPENAI_THINKING_REALTEXT, [OPENAI_GOOD], "deg-w2");
assert.equal(fetchCalls, 0, "non-whitespace visible text bars the retry — no false positive on a normal answer");
assert.ok(out.includes("Here is the answer"), "the normal answer passes through untouched");
});

// #732 follow-up review: the double-degenerate pass-through (per-request budget
// already spent, round still ended degenerate) is observable only through its
// warn line — pin both directions so a regression that drops or spams it fails
// here instead of silently vanishing from fleet logs.

test("#732-follow-up R1: double-degenerate pass-through emits exactly one DOUBLE-degenerate warn", async () => {
const lines: Array<{ level: string; msg: string }> = [];
setLogCapture((level, msg) => { lines.push({ level, msg }); });
try {
// Re-request degenerate → retry fires (spends the per-request budget);
// the retry comes back degenerate too → passes through with the warn.
const { fetchCalls } = await drain([ROUND_DEGENERATE, ROUND_DEGENERATE], "deg-r1");
assert.equal(fetchCalls, 2, "re-request + one retry");
} finally {
setLogCapture(null);
}
const warns = lines.filter((l) => l.level === "warn" && l.msg.includes("DOUBLE-degenerate"));
assert.equal(warns.length, 1, "exactly one double-degenerate warn");
assert.ok(
warns[0].msg.includes("retry budget was already spent"),
"the warn attributes the pass-through to the spent per-request budget",
);
});

test("#732-follow-up R2: a SUCCESSFUL retry emits no DOUBLE-degenerate warn", async () => {
const lines: Array<{ level: string; msg: string }> = [];
setLogCapture((level, msg) => { lines.push({ level, msg }); });
try {
const { fetchCalls } = await drain([ROUND_DEGENERATE, ROUND_GOOD], "deg-r2");
assert.equal(fetchCalls, 2, "re-request + one successful retry");
} finally {
setLogCapture(null);
}
assert.deepEqual(
lines.filter((l) => l.msg.includes("DOUBLE-degenerate")),
[],
"no double-degenerate warn when the retry recovered",
);
assert.ok(
lines.some((l) => l.level === "warn" && l.msg.includes("degenerate-turn auto-retry")),
"the auto-retry warn fired — the capture is live, so the negative above is meaningful",
);
});

// #1862: the degenerate-retry boundary on the Responses wire misfired in two ways.
// A: a completed reasoning+custom_tool_call turn carries executable output, but the
// adapter counted zero tool calls, so the gate treated it as degenerate and re-issued
Expand Down
Loading