From e7e38d4159e3d77d3f2ee4aa6c47911bde29afcd Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 03:30:14 +0800 Subject: [PATCH 1/4] fix(wiring): feed the loop lane's egress sends into the 429 governor (#4743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upstream 429 governor (upstreamThrottleLocalReject / noteUpstreamThrottle429 / noteUpstreamThrottleResolved) was wired only on the relay forwarding path. Three loop-lane chokepoints bypassed it, so a 429-storming upstream never armed the egress hold on the lane most prone to storms: - src/compress-loop-responses.ts — Responses-wire compress loop round-2+ resends - src/preflight.ts — preflight refold summary calls - src/reasoning-guard.ts — reasoning-guard continuation rounds Each now reports its verdict to the observe pair: a 429 arms the egress hold for the next client turn; a settled non-429 clears it. Observe-only (no local gate), matching the main-loop fix in #4725 — a mid-loop local reject would change re-request behavior under load. The observation split follows each site's fetch primitive: fetchWithRetry sites report at the settled boundary (a 429 exhausting the budget throws UpstreamHttpError(429)); fetchWithTimeout sites report per response, like relay. Adds tests/issue4743-loop-lane-429-observe.test.ts — three wiring pins that drive the real code path against a 429 upstream and assert the hold armed; they fail without the fix (verified: 0/3 pass on the unpatched tree). Closes #4743 --- src/compress-loop-responses.ts | 34 +-- src/preflight.ts | 4 + src/reasoning-guard.ts | 4 + tests/issue4743-loop-lane-429-observe.test.ts | 219 ++++++++++++++++++ 4 files changed, 248 insertions(+), 13 deletions(-) create mode 100644 tests/issue4743-loop-lane-429-observe.test.ts diff --git a/src/compress-loop-responses.ts b/src/compress-loop-responses.ts index 76f87c195b..fdabc70b95 100644 --- a/src/compress-loop-responses.ts +++ b/src/compress-loop-responses.ts @@ -16,6 +16,7 @@ import { type ResponseInputItem } from "acp-kernel/wire"; import { MAX_LOOP_ROUNDS } from "./loop/index.js"; import { stripResponsesText } from "./loop/tag-echo-filter.js"; import { fetchWithRetry, UpstreamHttpError } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved } from "./upstream-throttle.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { safePrefix, safeSuffix } from "./text-safe.js"; @@ -215,24 +216,31 @@ export async function compressLoopResponsesJson( requestBody.input = mergeAdjacentConfigurationUpdates(hoistTrappedToolItems(inputItems)); const roundBody = JSON.stringify(requestOptions.wireTransform ? requestOptions.wireTransform(requestBody) : requestBody); requestOptions.resign?.(requestOptions.headers, roundBody); - const result = await fetchWithRetry(requestOptions.url, { - method: "POST", - headers: requestOptions.headers, - body: roundBody, - ...(ctx.proxyUrl ? { dispatcher: proxyDispatcher(ctx.proxyUrl) } : {}), - }, undefined, undefined, (info) => { - // #189: correlate the rejection with the rewrite that preceded it. - const lc = lastCompressSuffix(ctx.session.lastCompress); - ctx.log(`[acp-proxy: responses upstream rejected replay (HTTP ${info.status}: ${info.detail.slice(0, 120)}); likely provider risk-control — retrying in ${info.delayMs}ms (attempt ${info.attempt}/${info.maxAttempts})${lc}]`); - loggerLog("warn", `[acp-compress-responses] upstream rejected replay (HTTP ${info.status}); retrying in ${info.delayMs}ms (attempt ${info.attempt}/${info.maxAttempts})${lc}`); - }).catch((e) => { + let result: Awaited>; + try { + result = await fetchWithRetry(requestOptions.url, { + method: "POST", + headers: requestOptions.headers, + body: roundBody, + ...(ctx.proxyUrl ? { dispatcher: proxyDispatcher(ctx.proxyUrl) } : {}), + }, undefined, undefined, (info) => { + // #189: correlate the rejection with the rewrite that preceded it. + const lc = lastCompressSuffix(ctx.session.lastCompress); + ctx.log(`[acp-proxy: responses upstream rejected replay (HTTP ${info.status}: ${info.detail.slice(0, 120)}); likely provider risk-control — retrying in ${info.delayMs}ms (attempt ${info.attempt}/${info.maxAttempts})${lc}]`); + loggerLog("warn", `[acp-compress-responses] upstream rejected replay (HTTP ${info.status}); retrying in ${info.delayMs}ms (attempt ${info.attempt}/${info.maxAttempts})${lc}`); + }); + // #4743: the Responses-wire compress loop feeds the 429 governor — a settled ok response clears any hold. + noteUpstreamThrottleResolved(requestOptions.url, requestOptions.headers, result.response.status); + } catch (e) { if (e instanceof UpstreamHttpError) { + // #4743: a terminal 429 exhausted the retry budget — arm the governor for the next client turn. + if (e.status === 429) noteUpstreamThrottle429(requestOptions.url, requestOptions.headers, null); const suffix = e.attempts > 1 ? ` after ${e.attempts} attempt(s)` : ""; ctx.log(`[acp-proxy: responses compress loop upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`); - loggerLog("error", `[acp-compress-responses] upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}`); + loggerLog("error", `[acp-compress-responses] upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`); } throw e; - }); + } try { current = await result.response.json() as Record; } finally { diff --git a/src/preflight.ts b/src/preflight.ts index 4c90bf5c26..55c254edc0 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -17,6 +17,7 @@ import { applyAbsorbView } from "./absorb.js"; import { adoptContentStore, ccrLoopConfig, contentStoreOf } from "./store.js"; import { applyRanges, normalizeRangeOrder, type RewriteCtx } from "./stream.js"; import { fetchWithTimeout, isTransientUpstreamError, replayMaxAttempts, replayBackoffMs, sleep, UpstreamHttpError } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved } from "./upstream-throttle.js"; import { dumpSummaryRejection } from "./error-dump.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { isTlsTrustCode, tlsTrustRemediation, TLS_TRUST_CODES } from "./upstream-fail.js"; @@ -1323,6 +1324,9 @@ async function requestSummaryBody(deps: PreflightDeps, body: string, stream: boo stage = "response body"; const text = await result.response.text(); deps.signal?.throwIfAborted(); + // #4743: the preflight refold feeds the 429 governor — a rate-limit 429 arms the hold (honouring Retry-After), any other settled response clears it. + if (result.response.status === 429) noteUpstreamThrottle429(deps.url, outHeaders, result.response.headers.get("retry-after")); + else noteUpstreamThrottleResolved(deps.url, outHeaders, result.response.status); if (!result.response.ok) throw new UpstreamHttpError(result.response.status, text, attempt); return text; } catch (err) { diff --git a/src/reasoning-guard.ts b/src/reasoning-guard.ts index 3196bf82ee..e070494e59 100644 --- a/src/reasoning-guard.ts +++ b/src/reasoning-guard.ts @@ -1,4 +1,5 @@ import { fetchWithTimeout } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved } from "./upstream-throttle.js"; import { scrubLoneSurrogatesOnWire } from "./text-safe.js"; import { normalizeSseLineEndings, finalizeSseLineEndings } from "./sse-util.js"; import { awaitDrain } from "./server/stream-io.js"; @@ -558,6 +559,9 @@ export async function runReasoningGuard(p: GuardParams): Promise { finish(buildIncomplete(st, "upstream_error")); return; } + // #4743: reasoning-guard continuation rounds feed the 429 governor — a 429 arms the hold for the next client turn. + if (roundRes.response.status === 429) noteUpstreamThrottle429(p.upstreamUrl, p.reqHeaders, roundRes.response.headers.get("retry-after")); + else noteUpstreamThrottleResolved(p.upstreamUrl, p.reqHeaders, roundRes.response.status); const contBody = roundRes.response.body; roundRes.stopIdleTimer(); result = contBody ? await consumeRound(contBody, st, p.res) : { error: "no_body" }; diff --git a/tests/issue4743-loop-lane-429-observe.test.ts b/tests/issue4743-loop-lane-429-observe.test.ts new file mode 100644 index 0000000000..a985a2bf67 --- /dev/null +++ b/tests/issue4743-loop-lane-429-observe.test.ts @@ -0,0 +1,219 @@ +// #4743 — the loop lane's own egress sends feed the 429 governor. +// +// The governor's observe pair (noteUpstreamThrottle429 / noteUpstreamThrottleResolved) +// was wired only on the relay forwarding path. Three loop-lane chokepoints still +// bypassed it: the Responses-wire compress loop, the preflight refold, and the +// reasoning-guard continuation rounds. Each of those sites now reports its verdict +// so a 429-storming upstream arms the egress hold for the NEXT client turn. +// +// Unlike PR #4725's governor-contract pin (which passes on master too), these are +// WIRING pins: they drive the real code path against a 429 upstream and assert the +// governor armed — so they FAIL on master and pass only once the site observes. +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { randomUUID } from "node:crypto"; +import { afterEach, test } from "node:test"; +import { createCore, defaultConfig, defaultPrompts, type Config, type CoreMessage } from "acp-kernel"; +import { compressLoopResponsesJson } from "../src/compress-loop-responses.ts"; +import { runReasoningGuard, type SseSink } from "../src/reasoning-guard.ts"; +import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; +import { getSession } from "../src/session.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { upstreamThrottleLocalReject, _resetUpstreamThrottleForTest } from "../src/upstream-throttle.ts"; + +process.env.NODE_ENV = "test"; +process.env.BILI_REPLAY_RETRY_MAX = "3"; +process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; +_setStoreForTest(new SessionStore({ enabled: false })); + +afterEach(() => { + _resetUpstreamThrottleForTest(); + process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; +}); + +test("responses-wire compress loop feeds the 429 governor (#4743)", async () => { + const baseCore = createCore(); + const core = { ...baseCore, applyCompression: (input: Parameters[0]) => ({ state: input.state, result: { blocksCreated: 0, tokensCompressed: 0, errors: [], warnings: [] } }) }; + const ctx = { + core, + config: { modelContextLimit: 200000 } as Config, + messages: [] as CoreMessage[], + session: getSession(`issue4743-rsp-${randomUUID()}`), + log: () => {}, + }; + const url = "https://loop-4743.test/responses"; + const headers: Record = { "content-type": "application/json" }; + const initial = { + id: "r4743", + status: "completed", + output: [{ type: "function_call", id: "fc1", call_id: "c1", name: "compress", arguments: JSON.stringify({ content: [{ startId: "m1", endId: "m1", summary: "s" }] }) }], + }; + const body = { model: "gpt-5", input: [{ type: "message", role: "user", content: "go" }] }; + + const prevFetch = globalThis.fetch; + try { + // Arm: the loop's round-2 resend hits a 429 upstream; fetchWithRetry exhausts + // its budget and throws UpstreamHttpError(429), whose observe arms the governor. + globalThis.fetch = (async () => new Response(JSON.stringify({ error: "rate_limited" }), { status: 429, headers: { "content-type": "application/json", "retry-after": "1" } })) as typeof fetch; + let threw = false; + try { + await compressLoopResponsesJson(initial, ctx, body, { url, headers }); + } catch { + threw = true; + } + assert.ok(threw, "a terminal 429 propagates out of the loop"); + assert.ok(upstreamThrottleLocalReject(url, headers), "governor armed after the loop's terminal 429"); + + // Clear: a settled ok response lifts the hold. + globalThis.fetch = (async () => new Response(JSON.stringify({ id: "ok", status: "completed", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "done" }] }] }), { status: 200, headers: { "content-type": "application/json" } })) as typeof fetch; + await compressLoopResponsesJson(initial, ctx, body, { url, headers }); + assert.equal(upstreamThrottleLocalReject(url, headers), undefined, "a resolved verdict lifts the hold"); + } finally { + globalThis.fetch = prevFetch; + } +}); + +// --- reasoning-guard helpers (mirrored from tests/reasoning-guard.test.ts) --- +function sseRound(o: { reasoningTokens: number; encrypted?: string; messageText?: string }): string { + const nonReasoning = o.messageText ? 10 : 0; + const outputTokens = o.reasoningTokens + nonReasoning; + const events: Array> = []; + events.push({ type: "response.created", response: { id: "resp_x", status: "in_progress", model: "gpt-5.5" } }); + events.push({ type: "response.output_item.added", output_index: 0, item: { id: "rs_1", type: "reasoning", status: "in_progress" } }); + const reasonDone: Record = { id: "rs_1", type: "reasoning", status: "completed" }; + if (o.encrypted !== undefined) reasonDone.encrypted_content = o.encrypted; + events.push({ type: "response.output_item.done", output_index: 0, item: reasonDone }); + let oi = 1; + if (o.messageText) { + events.push({ type: "response.output_item.added", output_index: oi, item: { id: "msg_1", type: "message", role: "assistant", status: "in_progress", content: [] } }); + events.push({ type: "response.output_item.done", output_index: oi, item: { id: "msg_1", type: "message", role: "assistant", status: "completed", content: [{ type: "output_text", text: o.messageText }] } }); + oi++; + } + events.push({ type: "response.completed", response: { id: "resp_x", status: "completed", model: "gpt-5.5", output: [], usage: { input_tokens: 100, output_tokens: outputTokens, total_tokens: 100 + outputTokens, output_tokens_details: { reasoning_tokens: o.reasoningTokens } } } }); + let out = ""; + for (const e of events) out += `data: ${JSON.stringify(e)}\n\n`; + out += "data: [DONE]\n\n"; + return out; +} + +function streamFrom(text: string): ReadableStream { + const enc = new TextEncoder(); + return new ReadableStream({ start(controller) { controller.enqueue(enc.encode(text)); controller.close(); } }); +} + +function makeSink(): { sink: SseSink } { + const sink: SseSink = { + headersSent: false, + writableEnded: false, + writeHead() { sink.headersSent = true; }, + write() { return true; }, + end() { sink.writableEnded = true; }, + once() { return undefined; }, + }; + return { sink }; +} + +test("reasoning-guard continuation rounds feed the 429 governor (#4743)", async () => { + const server = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(429, { "retry-after": "1" }); + res.end(); + }); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const addr = server.address(); + assert.ok(addr && typeof addr === "object"); + const url = `http://127.0.0.1:${addr.port}/v1/responses`; + const headers: Record = { "content-type": "application/json" }; + try { + // A lattice-truncated first round (516 reasoning tokens + encrypted_content) + // triggers a continuation round; that round's 429 verdict arms the governor. + const firstResponse = new Response(streamFrom(sseRound({ reasoningTokens: 516, encrypted: "enc1" })), { headers: { "content-type": "text/event-stream" } }); + const { sink } = makeSink(); + await runReasoningGuard({ + firstResponse, + clearFirstTimer: () => {}, + upstreamUrl: url, + reqHeaders: headers, + originalBody: JSON.stringify({ model: "gpt-5.5", stream: true, input: [] }), + signal: new AbortController().signal, + res: sink, + config: { enabled: true }, + log: () => {}, + }); + assert.ok(upstreamThrottleLocalReject(url, headers), "governor armed after a 429 continuation round"); + } finally { + const closed = once(server, "close"); + server.close(); + server.closeAllConnections?.(); + await closed; + } +}); + +const PREFLIGHT_SUMMARY = "SUMMARY: keep the goal and acceptance criteria."; + +function preflightFixture(url: string) { + const session = getSession(`issue4743-preflight-${randomUUID()}`); + session.metadata.preflightStreamSummary = true; + const messages: CoreMessage[] = [ + { id: "first", role: "user", contentType: "text", text: "Keep the task goal." }, + { id: "large", role: "assistant", contentType: "text", text: "FILLER_".repeat(4000) }, + { id: "last", role: "user", contentType: "text", text: "Continue." }, + ]; + const deps: PreflightDeps = { + core: createCore(), + session, + config: defaultConfig(6000, { preserveRecentMessages: 0, preserveRecentTokens: 0 }), + prompts: defaultPrompts, + protocol: "responses", + url, + headers: {}, + model: "test-model", + log: () => {}, + }; + return { deps, messages }; +} + +test("preflight refold feeds the 429 governor (#4743)", async () => { + let mode: "429" | "200" = "429"; + const server = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + if (mode === "429") { + res.writeHead(429, { "retry-after": "1" }); + res.end("rate limited"); + } else { + const event = { type: "response.completed", response: { status: "completed", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: PREFLIGHT_SUMMARY }] }] } }; + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(`data: ${JSON.stringify(event)}\n\n`); + } + }); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const addr = server.address(); + assert.ok(addr && typeof addr === "object"); + const url = `http://127.0.0.1:${addr.port}/responses`; + const headers: Record = {}; + try { + // Arm: the preflight summary call 429s (each retry attempt reports the verdict). + mode = "429"; + const armed = preflightFixture(url); + await preflightCompress(armed.deps, armed.messages); + assert.ok(upstreamThrottleLocalReject(url, headers), "governor armed after a 429 summary"); + + // Clear: a successful summary lifts the hold (same origin + credential key). + mode = "200"; + const cleared = preflightFixture(url); + await preflightCompress(cleared.deps, cleared.messages); + assert.equal(upstreamThrottleLocalReject(url, headers), undefined, "a resolved summary lifts the hold"); + } finally { + const closed = once(server, "close"); + server.close(); + server.closeAllConnections?.(); + await closed; + } +}); From 50b2bf859cd12e001f439dcf7c367d04cca8f0fa Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 03:55:20 +0800 Subject: [PATCH 2/4] fix(wiring): loop-lane boundary report clears the hold on any settled non-429 (#4743 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchWithRetry throws UpstreamHttpError for terminal non-ok verdicts; the boundary observation armed on 429 but stayed silent on e.g. a fail-fast 400, whereas the #4204 contract clears the hold on ANY resolved non-429 (the endpoint demonstrably processed a request) — relay parity, same as the sibling #4752 site. Transport deaths remain unobserved (circuit-breaker territory, #3148). The responses-wire pin gains a re-arm -> fail-fast-400 phase asserting the hold lifts (verified failing without the fix line). --- src/compress-loop-responses.ts | 3 ++- tests/issue4743-loop-lane-429-observe.test.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/compress-loop-responses.ts b/src/compress-loop-responses.ts index fdabc70b95..a52fc6efbe 100644 --- a/src/compress-loop-responses.ts +++ b/src/compress-loop-responses.ts @@ -233,8 +233,9 @@ export async function compressLoopResponsesJson( noteUpstreamThrottleResolved(requestOptions.url, requestOptions.headers, result.response.status); } catch (e) { if (e instanceof UpstreamHttpError) { - // #4743: a terminal 429 exhausted the retry budget — arm the governor for the next client turn. + // #4743: a terminal 429 exhausted the retry budget — arm the governor for the next client turn; any other settled status proves the endpoint is processing again — clear it (relay parity, #4204). if (e.status === 429) noteUpstreamThrottle429(requestOptions.url, requestOptions.headers, null); + else noteUpstreamThrottleResolved(requestOptions.url, requestOptions.headers, e.status); const suffix = e.attempts > 1 ? ` after ${e.attempts} attempt(s)` : ""; ctx.log(`[acp-proxy: responses compress loop upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`); loggerLog("error", `[acp-compress-responses] upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`); diff --git a/tests/issue4743-loop-lane-429-observe.test.ts b/tests/issue4743-loop-lane-429-observe.test.ts index a985a2bf67..fad6d61089 100644 --- a/tests/issue4743-loop-lane-429-observe.test.ts +++ b/tests/issue4743-loop-lane-429-observe.test.ts @@ -69,6 +69,18 @@ test("responses-wire compress loop feeds the 429 governor (#4743)", async () => globalThis.fetch = (async () => new Response(JSON.stringify({ id: "ok", status: "completed", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "done" }] }] }), { status: 200, headers: { "content-type": "application/json" } })) as typeof fetch; await compressLoopResponsesJson(initial, ctx, body, { url, headers }); assert.equal(upstreamThrottleLocalReject(url, headers), undefined, "a resolved verdict lifts the hold"); + + // Review fix (#4772): a fail-fast NON-429 verdict also settles the endpoint — + // the #4204 contract clears on ANY resolved non-429 (relay parity), so the + // loop's boundary report must cover thrown UpstreamHttpError(≠429) too. + globalThis.fetch = (async () => new Response(JSON.stringify({ error: "rate_limited" }), { status: 429 })) as typeof fetch; + try { await compressLoopResponsesJson(initial, ctx, body, { url, headers }); } catch { /* expected */ } + assert.ok(upstreamThrottleLocalReject(url, headers), "re-armed by a second terminal 429"); + globalThis.fetch = (async () => new Response(JSON.stringify({ error: "invalid model" }), { status: 400, headers: { "content-type": "application/json" } })) as typeof fetch; + let threw400 = false; + try { await compressLoopResponsesJson(initial, ctx, body, { url, headers }); } catch { threw400 = true; } + assert.ok(threw400, "a fail-fast 400 propagates out of the loop"); + assert.equal(upstreamThrottleLocalReject(url, headers), undefined, "a settled non-429 verdict lifts the hold"); } finally { globalThis.fetch = prevFetch; } From c1e0b726b7b6bbe398bcc86163f3a4b438e54443 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 06:03:28 +0800 Subject: [PATCH 3/4] fix(wiring): preflight 429 arms the hold at the withhold verdict, not per-response (#4743 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-response arm added by this PR in the preflight walk self-blocked: the fits-case forward runs AFTER preflight and passes the relay gate, so a summary-probe 429 armed the hold that then answered the SAME request's own forward locally — violating the #301 contract (payload fits + preflight 429 → still forwarded; CI red on tests/preflight-fail-fast.test.ts, reproduced locally: 429 !== 200). Fix: the walk reports resolved per response (settled non-429 clears); the arm moves to the caller's withhold verdict in server.ts (kind === upstream && status === 429 → noteUpstreamThrottle429 with null retry-after). Nothing is forwarded on a withheld request, so the armed hold only answers the NEXT turn's relay forward locally — exactly the #4743 ask. Doomed-413 exits stay un-armed (they spend no summary call / different evidence). Pins: new e2e in tests/preflight-fail-fast.test.ts (503 withhold arms → next small turn gets local 429 stage:forward, zero egress; FAILS without the arm line — verified). The issue4743 preflight pin now asserts the anti-self-block (no arm from a 429 probe alone) + cleared-on-200. --- src/preflight.ts | 7 ++- src/server.ts | 3 ++ tests/issue4743-loop-lane-429-observe.test.ts | 26 ++++++--- tests/preflight-fail-fast.test.ts | 53 ++++++++++++++++++- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/preflight.ts b/src/preflight.ts index 55c254edc0..46f239a411 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -17,7 +17,7 @@ import { applyAbsorbView } from "./absorb.js"; import { adoptContentStore, ccrLoopConfig, contentStoreOf } from "./store.js"; import { applyRanges, normalizeRangeOrder, type RewriteCtx } from "./stream.js"; import { fetchWithTimeout, isTransientUpstreamError, replayMaxAttempts, replayBackoffMs, sleep, UpstreamHttpError } from "./fetch-util.js"; -import { noteUpstreamThrottle429, noteUpstreamThrottleResolved } from "./upstream-throttle.js"; +import { noteUpstreamThrottleResolved } from "./upstream-throttle.js"; import { dumpSummaryRejection } from "./error-dump.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { isTlsTrustCode, tlsTrustRemediation, TLS_TRUST_CODES } from "./upstream-fail.js"; @@ -1324,9 +1324,8 @@ async function requestSummaryBody(deps: PreflightDeps, body: string, stream: boo stage = "response body"; const text = await result.response.text(); deps.signal?.throwIfAborted(); - // #4743: the preflight refold feeds the 429 governor — a rate-limit 429 arms the hold (honouring Retry-After), any other settled response clears it. - if (result.response.status === 429) noteUpstreamThrottle429(deps.url, outHeaders, result.response.headers.get("retry-after")); - else noteUpstreamThrottleResolved(deps.url, outHeaders, result.response.status); + // #4743 review fix (#4772): a settled non-429 summary response proves the endpoint processed again — clear any hold. A 429 must NOT arm here: this request's own forward runs AFTER preflight and passes the relay gate, so arming now would block our own forward of a fitting payload (#301 contract: fits + preflight 429 → still forward). The arm lives at the caller's withhold verdict (server.ts) instead. + if (result.response.status !== 429) noteUpstreamThrottleResolved(deps.url, outHeaders, result.response.status); if (!result.response.ok) throw new UpstreamHttpError(result.response.status, text, attempt); return text; } catch (err) { diff --git a/src/server.ts b/src/server.ts index 2dd74daee8..7edb8cd8a3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -19,6 +19,7 @@ import { fetchWithTimeout, upstreamTimeoutMs } from "./fetch-util.js"; import { getUpstreamConnectionStatus, resolveProxy, resolveProxyDecision, proxyDispatcher, type UpstreamProxyDecision } from "./upstream-proxy.js"; import { buildProxyFailureResponse, isExpectedFailFast, type SkippedProxyInfo } from "./upstream-fail.js"; import { getUpstreamAlerts } from "./upstream-alerts.js"; +import { noteUpstreamThrottle429 } from "./upstream-throttle.js"; import { hostIdForLog, maskHostPortForLog, setMaskHostsEnabled, maskUrlForLog, maskUrlsInText } from "./log-mask.js"; // Protocol codecs + the historical-image strip primitive live in the kernel now // (single source of truth shared with the omp/pi adapters): import from @@ -4032,6 +4033,8 @@ export async function preflightCompressIfNeeded( ffDetail += ` ${result.compressedRanges} range(s) folded this turn persist in the session — the next attempt starts from a smaller baseline and continues the fold${shortfall !== undefined ? ` (~${shortfall.toLocaleString("en-US")} tokens short of the target)` : ""}.`; } const ff = failFast(status, ffDetail, retryable, result.compressedRanges > 0 ? foldedCalibratedSize : undefined, result.rangesRemaining); + // #4743 review fix (#4772): the withhold verdict carries the 429 evidence — arm the egress hold HERE, not per-response inside the walk: the fits-case forward runs after preflight and must not be blocked by its own summary probe's 429 (#301). Nothing is forwarded on this request, so the armed hold only answers the NEXT turn's relay forward locally. No retry-after header is available at this layer — base-delay backoff applies. + if (f?.kind === "upstream" && f.status === 429) noteUpstreamThrottle429(upstreamUrl, headers, null); // (#4079) the #726 dead-end cooldown arming was removed with the whole // mechanism — see the block comment where the walk starts. The retry // contract lives entirely in this answer's retryable:false/status. diff --git a/tests/issue4743-loop-lane-429-observe.test.ts b/tests/issue4743-loop-lane-429-observe.test.ts index fad6d61089..07073aa58d 100644 --- a/tests/issue4743-loop-lane-429-observe.test.ts +++ b/tests/issue4743-loop-lane-429-observe.test.ts @@ -6,6 +6,11 @@ // reasoning-guard continuation rounds. Each of those sites now reports its verdict // so a 429-storming upstream arms the egress hold for the NEXT client turn. // +// #4772 review fix: the preflight site reports RESOLVED per response but arms ONLY +// at the caller's withhold verdict (server.ts) — arming per-response there would +// block the same request's own forward of a fitting payload (#301). The e2e pin for +// the withhold-time arm lives in tests/preflight-fail-fast.test.ts. +// // Unlike PR #4725's governor-contract pin (which passes on master too), these are // WIRING pins: they drive the real code path against a 429 upstream and assert the // governor armed — so they FAIL on master and pass only once the site observes. @@ -20,7 +25,7 @@ import { runReasoningGuard, type SseSink } from "../src/reasoning-guard.ts"; import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; import { getSession } from "../src/session.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; -import { upstreamThrottleLocalReject, _resetUpstreamThrottleForTest } from "../src/upstream-throttle.ts"; +import { noteUpstreamThrottle429, upstreamThrottleLocalReject, _resetUpstreamThrottleForTest } from "../src/upstream-throttle.ts"; process.env.NODE_ENV = "test"; process.env.BILI_REPLAY_RETRY_MAX = "3"; @@ -189,7 +194,7 @@ function preflightFixture(url: string) { return { deps, messages }; } -test("preflight refold feeds the 429 governor (#4743)", async () => { +test("preflight refold clears the 429 hold without self-arming (#4743/#4772)", async () => { let mode: "429" | "200" = "429"; const server = http.createServer((req, res) => { req.resume(); @@ -211,13 +216,20 @@ test("preflight refold feeds the 429 governor (#4743)", async () => { const url = `http://127.0.0.1:${addr.port}/responses`; const headers: Record = {}; try { - // Arm: the preflight summary call 429s (each retry attempt reports the verdict). + // Anti-self-block (#4772 review): a 429'd summary probe must NOT arm the hold — + // this request's own forward runs AFTER preflight and passes the relay gate, so + // arming here would block our own forward of a fitting payload (#301 contract). + // The arm lives at the caller's withhold verdict (server.ts; e2e pin in + // tests/preflight-fail-fast.test.ts). This phase FAILS on the per-response-arm + // tree and passes only with the verdict-time arm. mode = "429"; - const armed = preflightFixture(url); - await preflightCompress(armed.deps, armed.messages); - assert.ok(upstreamThrottleLocalReject(url, headers), "governor armed after a 429 summary"); + const probed = preflightFixture(url); + await preflightCompress(probed.deps, probed.messages); + assert.equal(upstreamThrottleLocalReject(url, headers), undefined, "a 429'd summary probe alone does not arm the hold (no self-block)"); - // Clear: a successful summary lifts the hold (same origin + credential key). + // Clear: a settled non-429 summary response lifts an existing hold (same origin key). + noteUpstreamThrottle429(url, headers, null); + assert.ok(upstreamThrottleLocalReject(url, headers), "setup: hold armed"); mode = "200"; const cleared = preflightFixture(url); await preflightCompress(cleared.deps, cleared.messages); diff --git a/tests/preflight-fail-fast.test.ts b/tests/preflight-fail-fast.test.ts index 36230f1028..72d41e675b 100644 --- a/tests/preflight-fail-fast.test.ts +++ b/tests/preflight-fail-fast.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; -import test from "node:test"; +import { afterEach, test } from "node:test"; process.env.NODE_ENV = "test"; // Fail fast on the very first 429 instead of the default 3 attempts with @@ -16,6 +16,9 @@ import { METADATA_FOLD_COVERAGE } from "../src/fold-reconcile.ts"; import { getSession } from "../src/session.ts"; import { normalizeUpstreamOrigin } from "../src/util.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { _resetUpstreamThrottleForTest, upstreamThrottleLocalReject } from "../src/upstream-throttle.ts"; + +afterEach(() => { _resetUpstreamThrottleForTest(); }); // #301: when preflight (overflow) compression cannot proceed — the summary // upstream is rate-limiting, or the compress budget is exhausted — the proxy @@ -224,6 +227,54 @@ test("e2e #301: overflow + summary upstream 429 → structured 503, over-window } }); +test("e2e #4743: preflight 429 withhold arms the egress hold — the next forward answers locally", async () => { + const calls: Call[] = []; + // The summarization endpoint 429s (non-stream); stream forwards would 200 — + // so any stream call reaching upstream proves the hold did NOT answer locally. + const upstream = makeUpstream429(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + const targetUrl = `http://127.0.0.1:${upstreamPort}/v1/messages`; + + try { + // Turn 1: over-window history, summary upstream 429s → 503 fail-fast, + // payload withheld. The withhold verdict arms the egress hold (#4743). + const r1 = await fetch(`http://127.0.0.1:${proxyPort}/bili/${targetUrl}`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "issue4743-hold-sess" }, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + }); + assert.equal(r1.status, 503, "fail-fast 503 when the summary upstream rate-limits"); + await r1.text(); + assert.ok(upstreamThrottleLocalReject(targetUrl, { "content-type": "application/json" }), "the 429 withhold arms the egress hold"); + + // Turn 2: a small fitting payload — the relay gate must answer LOCALLY + // (429 + stage:"forward") instead of storming the rate-limited upstream again. + const r2 = await fetch(`http://127.0.0.1:${proxyPort}/bili/${targetUrl}`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "issue4743-hold-sess" }, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hello" }] }), + }); + assert.equal(r2.status, 429, "the held origin answers locally while the hold is armed"); + const json2 = JSON.parse(await r2.text()) as { error?: { type?: string; stage?: string } }; + assert.equal(json2.error?.type, "rate_limit_error"); + assert.equal(json2.error?.stage, "forward"); + assert.equal(calls.filter((c) => c.stream).length, 0, "zero egress while held — nothing was forwarded upstream"); + } finally { + proxy.close(); + proxy.closeIdleConnections?.(); + await once(proxy, "close"); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close"); + } +}); + test("e2e #301: payload fits the window + preflight 429 → request still forwarded (no false positive)", async () => { const calls: Call[] = []; let streamCalls = 0; From 242aaae1753aaa2c3bb65e0ad75c7fc03b2e0117 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 13 Nov 2026 10:06:10 +0800 Subject: [PATCH 4/4] fix: feed the non-stream Anthropic-wire compress loop into the 429 governor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix for #4772: the PR wired the Responses-wire compress loop, the preflight refold, and the reasoning-guard continuation rounds into the 429 governor, but missed the structurally-identical non-stream Anthropic-wire compress loop (compress-loop-anthropic.ts) — its round-2+ resends bypassed the governor, so a 429-storming upstream on that lane never armed the egress hold for the next client turn. Mirrors the Responses-wire observe pair: a settled ok response clears the hold; a terminal 429 arms it; any other settled non-429 clears it (relay parity, #4204); transport death stays unobserved (circuit-breaker territory). Adds a wiring pin (tests/issue4743-loop-lane-429-observe.test.ts) that drives the real loop against a 429 upstream and asserts the governor armed — fails on the pre-fix tree, passes with the fix. --- src/compress-loop-anthropic.ts | 27 +++++++---- tests/issue4743-loop-lane-429-observe.test.ts | 45 +++++++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/compress-loop-anthropic.ts b/src/compress-loop-anthropic.ts index 3f85a4888c..e23f7c37dc 100644 --- a/src/compress-loop-anthropic.ts +++ b/src/compress-loop-anthropic.ts @@ -47,6 +47,7 @@ import { isProxyToolFor } from "./absorb.js"; import { withSessionLock } from "./session.js"; import { drainPendingRetrievals } from "./store.js"; import { fetchWithRetry, UpstreamHttpError } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved } from "./upstream-throttle.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { COMPRESS_TOOL_NAME } from "./compress-tool.js"; import { applyConfiguredCompression } from "./external-summary-compress.js"; @@ -147,15 +148,25 @@ export async function compressLoopAnthropicJson( if (requestOptions.egress) roundBody = await requestOptions.egress(roundBody); requestOptions.resign?.(requestOptions.headers, roundBody); log(`[acp-proxy: non-stream anthropic proxy-tool round ${round}; re-requesting so the model sees the result]`); - const result = await fetchWithRetry( - requestOptions.url, - { method: "POST", headers: requestOptions.headers, body: roundBody, ...(options.proxyUrl ? { dispatcher: proxyDispatcher(options.proxyUrl) } : {}) }, - undefined, - options.signal, - ).catch((e) => { - if (e instanceof UpstreamHttpError) log(`[acp-proxy: non-stream anthropic loop upstream error ${e.status}: ${e.body.slice(0, 200)}]`); + let result: Awaited>; + try { + result = await fetchWithRetry( + requestOptions.url, + { method: "POST", headers: requestOptions.headers, body: roundBody, ...(options.proxyUrl ? { dispatcher: proxyDispatcher(options.proxyUrl) } : {}) }, + undefined, + options.signal, + ); + // #4743: the non-stream Anthropic-wire compress loop feeds the 429 governor — a settled ok response clears any hold. + noteUpstreamThrottleResolved(requestOptions.url, requestOptions.headers, result.response.status); + } catch (e) { + if (e instanceof UpstreamHttpError) { + // #4743: a terminal 429 exhausted the retry budget — arm the governor; any other settled status clears it (relay parity, #4204). + if (e.status === 429) noteUpstreamThrottle429(requestOptions.url, requestOptions.headers, null); + else noteUpstreamThrottleResolved(requestOptions.url, requestOptions.headers, e.status); + log(`[acp-proxy: non-stream anthropic loop upstream error ${e.status}: ${e.body.slice(0, 200)}]`); + } throw e; - }); + } try { current = (await result.response.json()) as Record; } finally { diff --git a/tests/issue4743-loop-lane-429-observe.test.ts b/tests/issue4743-loop-lane-429-observe.test.ts index 8088de9bda..f1b3f9c372 100644 --- a/tests/issue4743-loop-lane-429-observe.test.ts +++ b/tests/issue4743-loop-lane-429-observe.test.ts @@ -21,6 +21,8 @@ import { randomUUID } from "node:crypto"; import { afterEach, test } from "node:test"; import { createCore, defaultConfig, defaultPrompts, type Config, type CoreMessage } from "acp-kernel"; import { compressLoopResponsesJson } from "../src/compress-loop-responses.ts"; +import { compressLoopAnthropicJson } from "../src/compress-loop-anthropic.ts"; +import type { RewriteCtx } from "../src/stream.ts"; import { runReasoningGuard, type SseSink } from "../src/reasoning-guard.ts"; import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; import { getSession } from "../src/session.ts"; @@ -91,6 +93,49 @@ test("responses-wire compress loop feeds the 429 governor (#4743)", async () => } }); +test("non-stream anthropic-wire compress loop feeds the 429 governor (#4743)", async () => { + const baseCore = createCore(); + const core = { ...baseCore, applyCompression: (input: Parameters[0]) => ({ state: input.state, result: { blocksCreated: 0, tokensCompressed: 0, errors: [], warnings: [] } }) }; + const ctx: RewriteCtx = { + core, + config: { modelContextLimit: 200000 } as Config, + messages: [] as CoreMessage[], + session: getSession(`issue4743-anth-${randomUUID()}`), + log: () => {}, + }; + const url = "https://loop-4743.test/anthropic"; + const headers: Record = { "content-type": "application/json" }; + const initial = { + id: "msg_4743", + type: "message", + role: "assistant", + content: [{ type: "tool_use", id: "tu1", name: "compress", input: { content: [{ startId: "m1", endId: "m1", summary: "s" }] } }], + }; + const makeBody = () => ({ model: "claude-sonnet", messages: [{ role: "user", content: "go" }] }); + + const prevFetch = globalThis.fetch; + try { + // Arm: the loop's round-2 resend hits a 429 upstream; fetchWithRetry exhausts + // its budget and throws UpstreamHttpError(429), whose observe arms the governor. + globalThis.fetch = (async () => new Response(JSON.stringify({ error: "rate_limited" }), { status: 429, headers: { "content-type": "application/json", "retry-after": "1" } })) as typeof fetch; + let threw = false; + try { + await compressLoopAnthropicJson(initial, ctx, { signal: new AbortController().signal }, makeBody(), { url, headers }); + } catch { + threw = true; + } + assert.ok(threw, "a terminal 429 propagates out of the loop"); + assert.ok(upstreamThrottlePeekReject(url, headers), "governor armed after the loop's terminal 429"); + + // Clear: a settled ok response lifts the hold (same origin key). + globalThis.fetch = (async () => new Response(JSON.stringify({ id: "msg_ok", type: "message", role: "assistant", content: [{ type: "text", text: "done" }] }), { status: 200, headers: { "content-type": "application/json" } })) as typeof fetch; + await compressLoopAnthropicJson(initial, ctx, { signal: new AbortController().signal }, makeBody(), { url, headers }); + assert.equal(upstreamThrottlePeekReject(url, headers), undefined, "a resolved verdict lifts the hold"); + } finally { + globalThis.fetch = prevFetch; + } +}); + // --- reasoning-guard helpers (mirrored from tests/reasoning-guard.test.ts) --- function sseRound(o: { reasoningTokens: number; encrypted?: string; messageText?: string }): string { const nonReasoning = o.messageText ? 10 : 0;