From 66f170c72278deb5792cc2645db1bde9ba57d8d8 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 26 Oct 2026 00:35:56 +0800 Subject: [PATCH 1/2] fix(wiring): degenerate-turn continuation retry joins the 429 governor (#4743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry is an egress send: observe-side feeds noteUpstreamThrottle429/ Resolved like any main-lane send (one observation per request), and the gate answers locally under an armed hold instead of burning an upstream slot — passing the empty turn through preserves the client's stream. --- src/degenerate-retry.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/degenerate-retry.ts b/src/degenerate-retry.ts index ba3ea55498..a21b28bd73 100644 --- a/src/degenerate-retry.ts +++ b/src/degenerate-retry.ts @@ -1,4 +1,5 @@ import { fetchWithTimeout } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved, upstreamThrottleLocalReject } from "./upstream-throttle.js"; import { hasCompactionTrigger } from "./codex-compact.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import type { WireProtocol } from "./util.js"; @@ -82,6 +83,17 @@ export function makeContinuationRefetch(opts: ContinuationRetryOpts): () => Prom const wireRetryBody = opts.wrapBody ? opts.wrapBody(retryBody) : retryBody; try { opts.resign?.(opts.reqHeaders, wireRetryBody); + // #4743: the retry is an egress send like any other — feed the 429 + // governor (observe only, no local gate: the retry is bounded by + // the degenerate-turn ladder, and holding IT locally would strand + // the client's already-open response stream). A 429 verdict arms + // the hold for the NEXT main-lane send; any other resolved status + // clears it — same one-observation-per-request discipline as relay. + const throttleReject = upstreamThrottleLocalReject(opts.upstreamUrl, opts.reqHeaders); + if (throttleReject !== undefined) { + opts.log("info", `[${opts.label}] [plugin] degenerate-terminal retry answered locally by the 429 hold (#4204) — passing the empty turn through`); + return null; + } const r = await fetchWithTimeout( opts.upstreamUrl, { @@ -95,9 +107,12 @@ export function makeContinuationRefetch(opts: ContinuationRetryOpts): () => Prom ); if (!r.response.ok || !r.response.body) { r.clearTimer(); + if (r.response.status === 429) noteUpstreamThrottle429(opts.upstreamUrl, opts.reqHeaders, r.response.headers.get("retry-after")); + else noteUpstreamThrottleResolved(opts.upstreamUrl, opts.reqHeaders, r.response.status); opts.log("warn", `[${opts.label}] [plugin] degenerate-terminal retry rejected (HTTP ${r.response.status}); passing the empty turn through`); return null; } + noteUpstreamThrottleResolved(opts.upstreamUrl, opts.reqHeaders, r.response.status); const body = r.response.body as ReadableStream; // The retry stream is spliced into a response the caller already // owns, so its own idle timer is dropped — a stalled retry then ends From 795cd6d398c1f569fced40c1ca691e8f9334db0c Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 26 Oct 2026 12:43:12 +0800 Subject: [PATCH 2/2] test(#4743): pin the degenerate-refetch 429 governor wiring (RED on master) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pins for ab6a7f10 (cherry-picked from #4752): (A) an armed hold answers the continuation refetch locally — zero upstream hits, empty turn passes through; (B) the refetch's own 429 verdict arms the governor and a resolved verdict lifts it. Both verified RED on master, green with the fix. --- .../issue4743-degenerate-refetch-429.test.ts | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/issue4743-degenerate-refetch-429.test.ts diff --git a/tests/issue4743-degenerate-refetch-429.test.ts b/tests/issue4743-degenerate-refetch-429.test.ts new file mode 100644 index 0000000000..675d6ae304 --- /dev/null +++ b/tests/issue4743-degenerate-refetch-429.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { makeContinuationRefetch } from "../src/degenerate-retry.ts"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved, upstreamThrottleLocalReject } from "../src/upstream-throttle.ts"; + +// #4743 (PR #4725 round-3 residue, cherry-picked from #4752 ab6a7f10): the +// degenerate-turn continuation refetch is an egress send like any other, but +// it was invisible to the 429 governor — a 429-storming upstream never armed +// the hold from retry traffic, and an already-armed hold still burned an +// upstream slot on the retry. The refetch now (a) answers locally under an +// armed hold (passing the empty turn through — holding the CLIENT's open +// stream is not an option), and (b) feeds noteUpstreamThrottle429 / +// noteUpstreamThrottleResolved like every other egress send. +// +// Both assertions are RED on master (no gate call, no notes): the hold case +// records a real server hit on master, and the 429 case leaves the governor +// un-armed. + +const CHAT_BODY = JSON.stringify({ model: "m", messages: [{ role: "user", content: "hi" }] }); + +function opts(url: string, label: string) { + return { + protocol: "openai" as const, + body: CHAT_BODY, + upstreamUrl: url, + reqHeaders: {}, + signal: new AbortController().signal, + log: () => {}, + label, + }; +} + +async function recordedServer(status: number, body: string): Promise<{ url: string; hits: () => number; close: () => Promise }> { + let hitCount = 0; + const server = http.createServer((_req, res) => { + hitCount++; + res.writeHead(status, { "content-type": status === 200 ? "text/event-stream" : "application/json" }); + res.end(body); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + if (typeof addr !== "object" || addr === null) throw new Error("no addr"); + return { + url: `http://127.0.0.1:${addr.port}/v1/chat/completions`, + hits: () => hitCount, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +test("#4743 A: an armed 429 hold answers the continuation refetch locally — the empty turn passes through and the upstream records ZERO hits", async () => { + const srv = await recordedServer(200, "event: message_start\ndata: {}\n\n"); + try { + // arm the governor for this exact upstream (same streak shape as the + // loop-lane contract test: two consecutive verdicts) + noteUpstreamThrottle429(srv.url, {}, null); + noteUpstreamThrottle429(srv.url, {}, null); + assert.ok(upstreamThrottleLocalReject(srv.url, {}), "precondition: hold armed"); + + const refetch = makeContinuationRefetch(opts(srv.url, "gate-4743")); + const out = await refetch(); + assert.equal(out, null, "the refetch must be answered locally (null = empty turn passes through)"); + assert.equal(srv.hits(), 0, "the armed hold must NOT burn an upstream slot on the retry"); + } finally { + await srv.close(); + noteUpstreamThrottleResolved(srv.url, {}, 200); + } +}); + +test("#4743 B: the refetch's own 429 verdict arms the governor; a resolved retry lifts the hold", async () => { + const rejected = await recordedServer(429, JSON.stringify({ error: { message: "rate limited" } })); + const ok = await recordedServer(200, "event: message_start\ndata: {}\n\n"); + try { + assert.equal(upstreamThrottleLocalReject(rejected.url, {}), undefined, "precondition: no hold yet"); + + const first = await makeContinuationRefetch(opts(rejected.url, "storm-4743"))(); + assert.equal(first, null, "a rejected retry passes the empty turn through"); + + const armed = upstreamThrottleLocalReject(rejected.url, {}); + assert.ok(armed, "the refetch's 429 verdict must reach the governor (arm for the next main-lane send)"); + + const second = await makeContinuationRefetch(opts(ok.url, "lift-4743"))(); + assert.ok(second, "an ok retry still yields its stream"); + // the ok settle resolved the (different-host) governor key of its own + // upstream; verify the observe pair by checking the rejected host's + // hold is still armed until an explicit resolved verdict lifts it + assert.ok(upstreamThrottleLocalReject(rejected.url, {}), "hold on the storming host persists until a resolved verdict"); + noteUpstreamThrottleResolved(rejected.url, {}, 200); + assert.equal(upstreamThrottleLocalReject(rejected.url, {}), undefined, "resolved verdict lifts the hold"); + } finally { + await rejected.close(); + await ok.close(); + } +});