diff --git a/src/acp-status.ts b/src/acp-status.ts index ae4fb707d8..3251886c1a 100644 --- a/src/acp-status.ts +++ b/src/acp-status.ts @@ -11,6 +11,8 @@ import { type CompressibleRange, } from "acp-kernel"; import { conflictClientOf, conflictEventsOf, formatConflictSection } from "./conflict-watch.js"; +import { getOpenUpstreamCircuits } from "./upstream-alerts.js"; +import { dshLedgerFloorTokens } from "./server/budget.js"; import { getBlindTunnelStats } from "./mitm.js"; import { getUnrecognizedPathStats } from "./server/observability.js"; import { ccrEnabled, ccrLoopConfig, contentStoreOf } from "./store.js"; @@ -226,6 +228,30 @@ export function handleAcpStatus(args: Record, ctx: AcpStatusCtx extra.push(`FOLD SUBSTRATE INVALID — last compress failure was attributed to ${lastCause}. The Compressible-ranges list above was re-derived from the current view; if folding a listed range fails again, start a fresh conversation — the folded base no longer matches what the client resends.`); } } + // #3148: upstream-failure state used to live only in bili.log — surface the + // circuit, the emergency-shrink baseline and the sticky planning-floor clamp + // here so hosts/models can see WHY rounds fail or output is capped. + // Display-only; none of this touches a wire decision. + const openCircuits = getOpenUpstreamCircuits(); + if (openCircuits.length > 0) { + for (const c of openCircuits.slice(0, 4)) { + extra.push(""); + extra.push(`UPSTREAM CIRCUIT OPEN — ${c.host}: ${c.consecutiveFailures} consecutive pre-response failures since ${c.openedAt !== null ? new Date(c.openedAt).toISOString() : "?"} · last failure ${c.lastFailure ? `${c.lastFailure.kind}${c.lastFailure.detail ? ` (${c.lastFailure.detail})` : ""} @ ${new Date(c.lastFailure.at).toISOString()}` : "?"}; next probe ${c.nextProbeAt !== null ? new Date(c.nextProbeAt).toISOString() : "soon"}. Requests to this host fail fast until the probe succeeds.`); + } + if (openCircuits.length > 4) extra.push(`UPSTREAM CIRCUIT OPEN — (+${openCircuits.length - 4} more host(s))`); + } + const shrinkAt = ctx.session.stats.emergencyShrinkAt; + if (typeof shrinkAt === "number") { + const ageMin = Math.max(0, Math.round((Date.now() - shrinkAt) / 60000)); + extra.push(""); + extra.push(`EMERGENCY SHRINK ARMED — baseline raised to local estimate ${ctx.session.stats.emergencyShrinkEstimate ?? "?"} tokens after a failed round with no usage report (${ageMin}m ago); truncation band active until the next real usage report.`); + } + const ledgerFloor = dshLedgerFloorTokens(ctx.session.metadata); + const lastClamp = ctx.session.stats.lastOutputClamp; + if (ledgerFloor > 0 || lastClamp !== undefined) { + extra.push(""); + extra.push(`OUTPUT BUDGET CLAMP —${lastClamp !== undefined ? ` last: max_tokens ${lastClamp.raw} -> ${lastClamp.capped} (window ${lastClamp.window}, input~${Math.round(lastClamp.planningInput)}) @ ${new Date(lastClamp.at).toISOString()}` : ""} · planning floor ${ledgerFloor} tok from the refused dsh compaction replay (#2490, sticky for the session lifetime). Only compression can recover the input; this clamp guarantees native-window fit.`); + } // #1097: the processTurn above already resolved the envelope when armed // (contentStoreOf is idempotent); when disarmed skip the disk read. const ccrArmed = ccrEnabled(ctx.session); diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index b89b4299d0..45b0608fad 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -1336,6 +1336,10 @@ export function settleUsageReport( // compaction boundaries via resetSessionCompression (session.ts). session.stats.lastUsageGradeTokens = session.stats.lastInputTokens; session.stats.lastInputTokensSource = "usage"; + // #3148: a real usage report supersedes the failed-round arm — drop the + // visibility marker so acp_status stops advertising a stale emergency state. + delete session.stats.emergencyShrinkAt; + delete session.stats.emergencyShrinkEstimate; session.stats.contextTokens = session.stats.lastInputTokens; session.stats.contextTokensSource = "usage"; if (session.metadata) session.metadata.contextTokensAt = Date.now(); diff --git a/src/server/budget.ts b/src/server/budget.ts index 95d9cc5676..78b89a5e1d 100644 --- a/src/server/budget.ts +++ b/src/server/budget.ts @@ -336,10 +336,14 @@ export function clampOutgoingOutput( * planning basis only ever tightens the cap (fail-closed direction). */ ledgerFloorTokens?: number; /** #2122/#1933 F1: per-route estimator calibration — scales the local-est - * side of estimateInputTokens so the clamp judges the payload on the same - * provider-billed scale as preflight/nudge. Absent → legacy raw behavior. */ kFactor?: number; kOrigin?: string; origin?: string }, + * side of estimateInputTokens so the clamp judges the payload on the same + * provider-billed scale as preflight/nudge. Absent → legacy raw behavior. */ kFactor?: number; kOrigin?: string; origin?: string }, sessionId: string, log: (level: string, msg: string) => void, + /** #3148: optional visibility hook — invoked with the exact clamp event when + * a cap is applied (callers record it on session.stats for acp_status). + * Never affects the wire decision. */ + onClamped?: (info: { raw: number; capped: number; window: number; planningInput: number; ledgerFloor: number }) => void, ): void { const raw = readOutputBudget(rebuilt, field); if (typeof raw !== "number") return; @@ -365,5 +369,6 @@ export function clampOutgoingOutput( // rescue right before the turn dies. const overTarget = ctx.headroomWindow !== undefined && ctx.headroomWindow < ctx.nativeWindow && planningInput >= ctx.headroomWindow; log("info", `[${sessionId}] output budget clamped ${raw} -> ${capped} (input~${inputEstimate}${ledgerFloor > inputEstimate ? `, planning floor ${ledgerFloor} from the refused dsh compaction replay (#2490)` : ""}, window=${ctx.nativeWindow}); prevents input+output overflow (#453)${overTarget ? `; input already exceeds the headroom-adjusted target ~${ctx.headroomWindow} that nudge/preflight enforce — only compression can recover it, this clamp guarantees native-window fit only (#2096)` : ""}`); + onClamped?.({ raw, capped, window: ctx.nativeWindow, planningInput, ledgerFloor }); } } diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index 4ef4edfb4d..0643c61c0a 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -302,7 +302,7 @@ export async function prepareGoogle( } const rebuilt: GoogleRequestBody = { ...parsed, contents: rebuiltContents, tools: toolsOut, systemInstruction }; - clampOutgoingOutput(rebuilt as Record, "generationConfig.maxOutputTokens", { systemText: googleClientSystem, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "google", rebuilt, opts, upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log); + clampOutgoingOutput(rebuilt as Record, "generationConfig.maxOutputTokens", { systemText: googleClientSystem, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "google", rebuilt, opts, upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); // #532: title-gen side requests carry their own tiny system — skip them. if (!isTitleGen && googleOutboundSystem !== undefined) { session.metadata.systemPromptTokens = countSystemAndToolsTokens(googleOutboundSystem, toolsOut); diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 35cf6dc025..196668b208 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -357,7 +357,7 @@ export async function prepareOpenai( rebuiltMessages = normalizeStrictEchoReasoning(rebuiltMessages, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed)), log, sessionId); const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined }; warnReasoningPairs(rebuiltMessages, log, sessionId); - clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log); + clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); // prompt_cache_retention is an OpenAI-host-only cache directive; the dsh // launcher forces PI_CACHE_RETENTION=long (for the session-id // prompt_cache_key) which makes the client also emit it. Third-party diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index f134f29f0c..2022ccfce9 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -503,7 +503,7 @@ export async function prepareResponses( const rebuilt: ResponsesRequestBody = { ...parsed, input: rebuiltInput, tools: toolsOut }; warnResponsesReasoningPairs(Array.isArray(rebuiltInput) ? rebuiltInput : [], log, sessionId); if (!isCompactionTrigger) { - clampOutgoingOutput(rebuilt as Record, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log); + clampOutgoingOutput(rebuilt as Record, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); } // Route with the upstream THIS request goes to — session.meta.upstreamOrigin // is first-wins and would keep injecting pck toward a relay we switched diff --git a/src/server/relay.ts b/src/server/relay.ts index b326e3c515..c25ad59130 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -46,7 +46,7 @@ import { rewriteResponsesJsonResponseAsync } from "../stream-responses.js"; import { observeResponsesTerminalState } from "../stream-terminal.js"; import { rewriteJsonResponseAsync, type RewriteCtx } from "../stream.js"; import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe.js"; -import { clearUpstreamAlertsForHost, recordUpstreamAlert } from "../upstream-alerts.js"; +import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason } from "../upstream-alerts.js"; import { formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering.js"; @@ -98,6 +98,11 @@ function armFailureShrink(prepared: Prepared, log: (level: string, msg: string) if (est > s.stats.lastInputTokens) { s.stats.lastInputTokens = est; s.stats.lastInputTokensSource = "estimate"; + // #3148: the armed state used to be visible only in this warn line — + // record it on the session so acp_status can show it to the host/model. + // Cleared by the next real usage report (settleUsageReport). + s.stats.emergencyShrinkAt = Date.now(); + s.stats.emergencyShrinkEstimate = est; // The error path returns before forward()'s trailing markDirty — the // arm must schedule its OWN save or it is lost on restart. markDirty(s); @@ -498,6 +503,18 @@ export async function forward( // instead of blaming the rebuild. noteInboundMessages(prepared.session, prepared.originalMessages); } + // #3148: round-level breaker — while the endpoint is provably dead, fail + // fast WITHOUT contacting it instead of burning another full cold send + + // the whole in-request retry ladder per host-driven retry round. The + // thrown detail carries the actionable state (last failure + next probe) + // that the dispatch catch renders into the 502 body. + const circuitBlock = upstreamCircuitBlockReason(upstreamUrl); + if (circuitBlock !== undefined) { + log("warn", `[${prepared?.session.id ?? "passthrough"}] [acp-proxy] upstream circuit open — failing fast without contacting ${upstreamUrl}: ${circuitBlock}`); + if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "upstream circuit open", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate); + if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream circuit open"); + throw new Error(`upstream circuit open: ${circuitBlock}`); + } let upstreamResult: Awaited>; try { upstreamResult = await fetchWithTransportRetry(upstreamUrl, init, undefined, clientAbort.signal, (info) => { @@ -506,16 +523,19 @@ export async function forward( recordUpstreamConnection(upstreamUrl, proxyUrl); // #1682: any resolved response proves the host is reachable again. clearUpstreamAlertsForHost(upstreamUrl); + recordUpstreamCircuitSuccess(upstreamUrl); } catch (error) { recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy); recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined, skippedProxy); + const failureDetail = formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy); + recordUpstreamCircuitFailure(upstreamUrl, error, proxyUrl !== undefined, failureDetail); // #604: a network-level failure (socket reset, timeout abort) also never // reports usage — arm the emergency shrink like the 5xx branch below. if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "network failure", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate); // [#1343] no response means the attached full text never reached the model — // drop-and-log it (a corrective note surfaces on the next qualifying request). if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream network failure"); - throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy)}`, { cause: error }); + throw new Error(`upstream request failed: ${failureDetail}`, { cause: error }); } // #552 learn-on-failure: a converting upstream that rejects a role (codex // ≥0.153 sends "developer"; vLLM/SGLang-style backends answer 400 diff --git a/src/session.ts b/src/session.ts index dc64f7c0cf..ef629fb22f 100644 --- a/src/session.ts +++ b/src/session.ts @@ -141,6 +141,16 @@ export type Session = { * consumers (upward window self-heal, #496 overflow-evidence gate, * stale-limit retraction) treat absent as untrusted. */ lastInputTokensSource?: "usage" | "estimate" | "overflow-arm"; + /** #3148: emergency shrink armed by a failed round with no usage report + * (armFailureShrink) — when + the armed local estimate. Visibility for + * the state that used to live only in the warn line; acp_status renders + * it. Cleared by the next real usage report (settleUsageReport). */ + emergencyShrinkAt?: number; + emergencyShrinkEstimate?: number; + /** #3148: most recent outgoing output-budget clamp event — the #2490 + * planning-floor clamps were previously visible only in log lines. + * Display-only: no decision path reads it. */ + lastOutputClamp?: { raw: number; capped: number; window: number; planningInput: number; ledgerFloor: number; at: number }; /** #1110: one-shot emergency ceiling armed by an upstream context- * overflow 400 (server.ts overflow handler) — live evidence this * session cannot exceed that size. Kept SEPARATE from lastInputTokens diff --git a/src/upstream-alerts.ts b/src/upstream-alerts.ts index 8c4b863f73..82053f14f0 100644 --- a/src/upstream-alerts.ts +++ b/src/upstream-alerts.ts @@ -11,6 +11,7 @@ * The web UI renders whatever is here as a global banner via the existing * /__bili/overview poll; nothing else consumes this table. */ +import { log as loggerLog } from "./logger.js"; import { classifyUpstreamFailure, hintWithSkippedProxy, redactedSkippedProxy, type SkippedProxyInfo, type UpstreamFailureKind } from "./upstream-fail.js"; export interface UpstreamAlert { @@ -110,3 +111,148 @@ export function getUpstreamAlerts(): UpstreamAlert[] { export function _resetUpstreamAlertsForTest(): void { alerts = new Map(); } + +// ── Upstream failure circuit breaker (#3148) ────────────────────────────── +// +// The alert table above tells the WEB UI "host unreachable"; the breaker stops +// bili from AMPLIFYING the outage itself. A host-driven retry loop (dsh +// re-enters the turn after every 502) used to burn one full cold send plus the +// whole in-request retry ladder per round against a dead endpoint (observed: +// 19 rounds × 260KiB at ~12s cadence after the user's local gateway went +// down). After CIRCUIT_OPEN_AFTER consecutive FINAL pre-response failures for +// a host, further requests fail fast — zero bytes sent — with an actionable +// error naming the last failure and the next probe time. One half-open probe +// runs every CIRCUIT_PROBE_COOLDOWN_MS and closes the circuit on success +// (any resolved response, mirroring clearUpstreamAlertsForHost). +// +// Symmetric to the compress-loop breaker (stream.ts: 3 strikes, decay-based +// disarm) — deliberately NO config surface yet; knobs are owner-gated (#3148). + +interface UpstreamCircuitFailure { + kind: string; + detail: string; + at: number; +} + +interface UpstreamCircuit { + host: string; + /** Consecutive FINAL pre-response failures (resets on any resolved response). */ + consecutiveFailures: number; + openedAt: number | null; + nextProbeAt: number | null; + lastFailure: UpstreamCircuitFailure | null; +} + +const CIRCUIT_OPEN_AFTER = 3; +let circuitProbeCooldownMs = 60_000; +const MAX_CIRCUITS = 32; + +interface CircuitEntry { + state: UpstreamCircuit; + /** True while the single half-open probe is in flight — concurrent + * requests fail fast instead of stacking parallel probes. */ + probeInFlight: boolean; +} + +let circuits = new Map(); + +function blockReasonText(c: UpstreamCircuit, now: number): string { + const lf = c.lastFailure + ? ` · last failure ${c.lastFailure.kind}${c.lastFailure.detail ? ` (${c.lastFailure.detail})` : ""} @ ${new Date(c.lastFailure.at).toISOString()}` + : ""; + const waitS = Math.max(0, Math.ceil(((c.nextProbeAt ?? now) - now) / 1000)); + return `${c.host}: ${c.consecutiveFailures} consecutive pre-response failures since ${new Date(c.openedAt ?? now).toISOString()}${lf}; next probe ${new Date(c.nextProbeAt ?? now).toISOString()} (~${waitS}s) — endpoint unreachable, stop resending full context`; +} + +/** Consult before sending to `url`. Returns undefined when the request may + * proceed (possibly AS the single half-open probe), or a human-readable + * reason when it must fail fast without contacting the endpoint. */ +export function upstreamCircuitBlockReason(url: string, now: number = Date.now()): string | undefined { + const entry = circuits.get(hostOf(url)); + if (!entry || entry.state.openedAt === null) return undefined; + const c = entry.state; + if (now < (c.nextProbeAt ?? 0)) return blockReasonText(c, now); + if (entry.probeInFlight) return blockReasonText(c, now); + entry.probeInFlight = true; + return undefined; +} + +/** Record one FINAL pre-response transport failure (the error escaped the + * in-request retry ladder). Same discipline as ALERT_KINDS: client aborts, + * post-connect timeouts and HTTP verdicts never COUNT toward opening — only + * "cannot reach" failures open the circuit. `detail` (already formatted by + * the caller) rides along so fail-fast errors stay actionable. While the + * circuit is OPEN this call is the half-open probe finishing without a + * response: the single-flight slot is released for ANY kind — an uncounted + * result (timeout/abort/unknown) proves neither liveness nor death, but + * holding the slot would pin the circuit open until process restart. */ +export function recordUpstreamCircuitFailure(url: string, error: unknown, viaProxy: boolean, detail?: string): void { + const kind = classifyUpstreamFailure(error, { viaProxy }); + const host = hostOf(url); + const now = Date.now(); + let entry = circuits.get(host); + if (!entry) { + if (!ALERT_KINDS.has(kind)) return; + if (circuits.size >= MAX_CIRCUITS) { + let oldestKey: string | undefined; + let oldestAt = Infinity; + for (const [k, e] of circuits) { + const t = e.state.lastFailure?.at ?? e.state.openedAt ?? Infinity; + if (t < oldestAt) { + oldestAt = t; + oldestKey = k; + } + } + if (oldestKey !== undefined) circuits.delete(oldestKey); + } + entry = { state: { host, consecutiveFailures: 0, openedAt: null, nextProbeAt: null, lastFailure: null }, probeInFlight: false }; + circuits.set(host, entry); + } + const c = entry.state; + const shortDetail = detail && detail.length > 240 ? detail.slice(0, 240) + "…" : (detail ?? ""); + if (c.openedAt !== null) { + // Open: the half-open probe just finished without a response — release + // the slot and re-arm the cooldown regardless of kind (see above). + c.lastFailure = { kind, detail: shortDetail, at: now }; + entry.probeInFlight = false; + c.nextProbeAt = now + circuitProbeCooldownMs; + loggerLog("warn", `[acp-proxy] upstream circuit probe FAILED for ${host} (${kind}) — re-armed until ${new Date(c.nextProbeAt).toISOString()} (#3148)`); + return; + } + if (!ALERT_KINDS.has(kind)) return; + c.lastFailure = { kind, detail: shortDetail, at: now }; + c.consecutiveFailures += 1; + if (c.consecutiveFailures >= CIRCUIT_OPEN_AFTER) { + c.openedAt = now; + c.nextProbeAt = now + circuitProbeCooldownMs; + loggerLog("warn", `[acp-proxy] upstream circuit OPEN for ${host} after ${c.consecutiveFailures} consecutive pre-response failures (last: ${kind}) — requests to this host fail fast until a probe succeeds (next probe ${new Date(c.nextProbeAt).toISOString()}) (#3148)`); + } +} + +/** Any resolved response proves the path works again — close the circuit. */ +export function recordUpstreamCircuitSuccess(url: string): void { + const entry = circuits.get(hostOf(url)); + if (!entry) return; + if (entry.state.openedAt !== null) { + loggerLog("info", `[acp-proxy] upstream circuit CLOSED for ${entry.state.host} — a response arrived from the endpoint; normal retry ladder resumes (#3148)`); + } + circuits.delete(entry.state.host); +} + +/** Open circuits for status surfaces (acp_status / web UI), newest first. + * Closed hosts keep no entry. Fresh copies. */ +export function getOpenUpstreamCircuits(): UpstreamCircuit[] { + return [...circuits.values()] + .filter((e) => e.state.openedAt !== null) + .map((e) => ({ ...e.state })) + .sort((a, b) => (b.openedAt ?? 0) - (a.openedAt ?? 0)); +} + +export function _resetUpstreamCircuitsForTest(): void { + circuits = new Map(); + circuitProbeCooldownMs = 60_000; +} + +export function _setCircuitCooldownForTest(ms: number): void { + circuitProbeCooldownMs = ms; +} diff --git a/tests/issue3148-upstream-circuit.test.ts b/tests/issue3148-upstream-circuit.test.ts new file mode 100644 index 0000000000..c837aeba0a --- /dev/null +++ b/tests/issue3148-upstream-circuit.test.ts @@ -0,0 +1,235 @@ +// #3148: upstream-unavailable amplification — round-level circuit breaker + +// acp_status visibility (emergency shrink / output clamp / circuit state). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { CoreMessage } from "acp-kernel"; +import { createCore, createInitialState, defaultConfig } from "acp-kernel"; +import type { Session } from "../src/session.ts"; +import { handleAcpStatus } from "../src/acp-status.ts"; +import { + getOpenUpstreamCircuits, + recordUpstreamCircuitFailure, + recordUpstreamCircuitSuccess, + upstreamCircuitBlockReason, + _resetUpstreamCircuitsForTest, + _setCircuitCooldownForTest, +} from "../src/upstream-alerts.ts"; +import { clampOutgoingOutput } from "../src/server/budget.ts"; +import { settleUsageReport } from "../src/cache-ledger.ts"; + +const URL_A = "http://dead-host.invalid:8443/v1/chat/completions"; +const URL_B = "http://other-host.invalid/v1/chat/completions"; + +function refused(): Error { + return Object.assign(new Error("connect ECONNREFUSED 10.0.0.1:8443"), { code: "ECONNREFUSED" }); +} +function headersTimeout(): Error { + return Object.assign(new Error("request timed out while waiting for headers"), { code: "UND_ERR_HEADERS_TIMEOUT" }); +} +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +test("#3148: circuit opens after 3 consecutive pre-response failures and fast-fails until a probe succeeds", () => { + _resetUpstreamCircuitsForTest(); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "fresh host: no block"); + + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "2 consecutive failures: not open yet"); + + recordUpstreamCircuitFailure(URL_A, refused(), false); + const open = getOpenUpstreamCircuits(); + assert.equal(open.length, 1, "circuit open after the 3rd consecutive failure"); + assert.equal(open[0].host, new URL(URL_A).host); + assert.ok(open[0].consecutiveFailures >= 3); + + const reason = upstreamCircuitBlockReason(URL_A); + assert.ok(reason !== undefined, "blocked while open"); + assert.ok(reason.includes("dead-host.invalid"), "reason names the host"); + assert.ok(reason.includes("consecutive pre-response failures"), "reason states the cause"); + assert.ok(reason.includes("next probe"), "reason gives the probe ETA"); + assert.equal(upstreamCircuitBlockReason(URL_B), undefined, "other hosts unaffected"); + + recordUpstreamCircuitSuccess(URL_A); + assert.equal(getOpenUpstreamCircuits().length, 0, "success closes the circuit"); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "no block after close"); +}); + +test("#3148: consecutive counter resets on any resolved response mid-stream", () => { + _resetUpstreamCircuitsForTest(); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitSuccess(URL_A); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + assert.equal(getOpenUpstreamCircuits().length, 0, "flapping upstream never reaches 3 CONSECUTIVE failures"); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined); +}); + +test("#3148: non-transport failures (timeouts/aborts) never count toward the circuit", () => { + _resetUpstreamCircuitsForTest(); + for (let i = 0; i < 5; i++) recordUpstreamCircuitFailure(URL_A, headersTimeout(), false); + assert.equal(getOpenUpstreamCircuits().length, 0, "upstream-timeout is not in the count set"); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined); +}); + +test("#3148: half-open allows exactly one probe; a failed probe re-arms the cooldown", async () => { + _resetUpstreamCircuitsForTest(); + _setCircuitCooldownForTest(50); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + assert.ok(upstreamCircuitBlockReason(URL_A) !== undefined, "open immediately after the 3rd failure"); + + await sleep(70); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "cooldown elapsed: this request becomes THE probe"); + assert.ok(upstreamCircuitBlockReason(URL_A) !== undefined, "a concurrent request while the probe is in flight stays blocked"); + + recordUpstreamCircuitFailure(URL_A, refused(), false); + assert.ok(upstreamCircuitBlockReason(URL_A) !== undefined, "failed probe re-arms the cooldown"); + await sleep(70); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "second probe granted after the new cooldown"); + recordUpstreamCircuitSuccess(URL_A); + assert.equal(getOpenUpstreamCircuits().length, 0, "probe success closes the circuit"); +}); + +test("#3148 review: an UNCOUNTED probe failure (timeout/client abort) must not pin the circuit open", async () => { + _resetUpstreamCircuitsForTest(); + _setCircuitCooldownForTest(50); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + assert.ok(upstreamCircuitBlockReason(URL_A) !== undefined, "open after the 3rd failure"); + + await sleep(70); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "cooldown elapsed: this request becomes THE probe"); + // The probe dies with an uncounted kind — a degraded host that accepts TCP + // but never answers (headers timeout), or a client that aborts mid-probe. + // Pre-fix this left the single-flight slot held and the circuit failed fast + // FOREVER (until process restart), even after full recovery. + recordUpstreamCircuitFailure(URL_A, headersTimeout(), false); + assert.ok(getOpenUpstreamCircuits()[0].lastFailure?.kind === "upstream-timeout", "uncounted probe failure is visible"); + assert.ok(upstreamCircuitBlockReason(URL_A) !== undefined, "re-armed cooldown blocks immediately"); + await sleep(70); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "slot released: next probe granted after the re-armed cooldown"); + + const aborted = Object.assign(new Error("request aborted"), { name: "AbortError" }); + recordUpstreamCircuitFailure(URL_A, aborted, false); + assert.equal(getOpenUpstreamCircuits().length, 1, "an abort proves nothing — circuit stays open"); + await sleep(70); + assert.equal(upstreamCircuitBlockReason(URL_A), undefined, "abort also released the slot"); + recordUpstreamCircuitSuccess(URL_A); + assert.equal(getOpenUpstreamCircuits().length, 0, "probe success closes the circuit"); +}); + +test("#3148: clampOutgoingOutput records the exact clamp event without changing the wire decision", () => { + const rebuilt: Record = { max_tokens: 65536 }; + let seen: { raw: number; capped: number; window: number; planningInput: number; ledgerFloor: number } | undefined; + // Reproduces the reporter's exact numbers: floor 997348, window 1048576 → + // margin ceil(997348*0.05)=49868, cap floor(1048576-997348-49868)=1360. + clampOutgoingOutput( + rebuilt, + "max_tokens", + { + systemText: "", + tools: [], + processedMessages: [], + lastInputTokens: 97730, + lastInputTokensSource: "usage", + nativeWindow: 1048576, + imageTokens: 0, + ledgerFloorTokens: 997348, + }, + "t", + () => {}, + (info) => { seen = info; }, + ); + assert.deepEqual(seen, { raw: 65536, capped: 1360, window: 1048576, planningInput: 997348, ledgerFloor: 997348 }, "exact clamp event captured"); + assert.equal(rebuilt.max_tokens, 1360, "wire decision unchanged (same cap as before the hook)"); + + const healthy: Record = { max_tokens: 65536 }; + let healthyCalls = 0; + clampOutgoingOutput( + healthy, + "max_tokens", + { + systemText: "", + tools: [], + processedMessages: [], + lastInputTokens: 97730, + lastInputTokensSource: "usage", + nativeWindow: 1048576, + imageTokens: 0, + }, + "t", + () => {}, + () => { healthyCalls++; }, + ); + assert.equal(healthyCalls, 0, "no clamp event when nothing was clamped"); + assert.equal(healthy.max_tokens, 65536, "budget untouched when headroom suffices"); +}); + +function makeSession(overrides?: Partial & { metadata?: Record }): Session { + return { + id: "issue3148-test", + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 15000, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0, ...overrides }, + metadata: overrides?.metadata ?? {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function makeCtx(session: Session): { core: ReturnType; config: ReturnType; messages: CoreMessage[]; session: Session } { + const messages: CoreMessage[] = [ + { id: "raw_1", role: "user", contentType: "text", text: "hello" }, + { id: "raw_2", role: "assistant", contentType: "text", text: "hi" }, + ]; + return { core: createCore(), config: defaultConfig(200000), messages, session }; +} + +test("#3148: acp_status surfaces the open circuit, armed emergency shrink and the sticky clamp floor", () => { + _resetUpstreamCircuitsForTest(); + _setCircuitCooldownForTest(60_000); + const session = makeSession({ + emergencyShrinkAt: Date.now() - 60_000, + emergencyShrinkEstimate: 699777, + lastOutputClamp: { raw: 65536, capped: 1360, window: 1048576, planningInput: 997348, ledgerFloor: 997348, at: Date.now() - 30_000 }, + }); + session.metadata.dshCompactionRefusedBytes = 997348 * 4; + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + recordUpstreamCircuitFailure(URL_A, refused(), false); + + const text = handleAcpStatus({}, makeCtx(session)).text; + assert.ok(text.includes("UPSTREAM CIRCUIT OPEN"), "open circuit visible"); + assert.ok(text.includes("dead-host.invalid"), "circuit names the host"); + assert.ok(text.includes("EMERGENCY SHRINK ARMED"), "armed shrink visible"); + assert.ok(text.includes("699777"), "shrink estimate visible"); + assert.ok(text.includes("OUTPUT BUDGET CLAMP"), "clamp section visible"); + assert.ok(text.includes("65536 -> 1360"), "last clamp values visible"); + assert.ok(text.includes("997348"), "planning floor visible"); +}); + +test("#3148: acp_status stays quiet about all three when none is active", () => { + _resetUpstreamCircuitsForTest(); + const text = handleAcpStatus({}, makeCtx(makeSession())).text; + assert.ok(!text.includes("UPSTREAM CIRCUIT OPEN"), "no circuit line when closed"); + assert.ok(!text.includes("EMERGENCY SHRINK ARMED"), "no shrink line when not armed"); + assert.ok(!text.includes("OUTPUT BUDGET CLAMP"), "no clamp line without ledger/clamp history"); +}); + +test("#3148: a real usage report clears the emergency-shrink visibility marker", () => { + _resetUpstreamCircuitsForTest(); + const session = makeSession({ emergencyShrinkAt: Date.now(), emergencyShrinkEstimate: 699777, lastInputTokensSource: "estimate" }); + settleUsageReport(session, { total: 1000, reportedCached: 0 }); + assert.equal(session.stats.emergencyShrinkAt, undefined, "marker cleared"); + assert.equal(session.stats.emergencyShrinkEstimate, undefined, "estimate cleared"); + assert.equal(session.stats.lastInputTokensSource, "usage", "baseline back to usage grade"); +});