Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/acp-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import {
type CompressibleRange,
} from "acp-kernel";
import { conflictClientOf, conflictEventsOf, formatConflictSection } from "./conflict-watch.js";
import { getOpenUpstreamCircuits } from "./upstream-alerts.js";
import { dshLedgerFloorTokens } from "./server/budget.js";
import { getBlindTunnelStats } from "./mitm.js";
import { getUnrecognizedPathStats } from "./server/observability.js";
import { ccrEnabled, ccrLoopConfig, contentStoreOf } from "./store.js";
Expand Down Expand Up @@ -226,6 +228,30 @@ export function handleAcpStatus(args: Record<string, unknown>, ctx: AcpStatusCtx
extra.push(`FOLD SUBSTRATE INVALID — last compress failure was attributed to ${lastCause}. The Compressible-ranges list above was re-derived from the current view; if folding a listed range fails again, start a fresh conversation — the folded base no longer matches what the client resends.`);
}
}
// #3148: upstream-failure state used to live only in bili.log — surface the
// circuit, the emergency-shrink baseline and the sticky planning-floor clamp
// here so hosts/models can see WHY rounds fail or output is capped.
// Display-only; none of this touches a wire decision.
const openCircuits = getOpenUpstreamCircuits();
if (openCircuits.length > 0) {
for (const c of openCircuits.slice(0, 4)) {
extra.push("");
extra.push(`UPSTREAM CIRCUIT OPEN — ${c.host}: ${c.consecutiveFailures} consecutive pre-response failures since ${c.openedAt !== null ? new Date(c.openedAt).toISOString() : "?"} · last failure ${c.lastFailure ? `${c.lastFailure.kind}${c.lastFailure.detail ? ` (${c.lastFailure.detail})` : ""} @ ${new Date(c.lastFailure.at).toISOString()}` : "?"}; next probe ${c.nextProbeAt !== null ? new Date(c.nextProbeAt).toISOString() : "soon"}. Requests to this host fail fast until the probe succeeds.`);
}
if (openCircuits.length > 4) extra.push(`UPSTREAM CIRCUIT OPEN — (+${openCircuits.length - 4} more host(s))`);
}
const shrinkAt = ctx.session.stats.emergencyShrinkAt;
if (typeof shrinkAt === "number") {
const ageMin = Math.max(0, Math.round((Date.now() - shrinkAt) / 60000));
extra.push("");
extra.push(`EMERGENCY SHRINK ARMED — baseline raised to local estimate ${ctx.session.stats.emergencyShrinkEstimate ?? "?"} tokens after a failed round with no usage report (${ageMin}m ago); truncation band active until the next real usage report.`);
}
const ledgerFloor = dshLedgerFloorTokens(ctx.session.metadata);
const lastClamp = ctx.session.stats.lastOutputClamp;
if (ledgerFloor > 0 || lastClamp !== undefined) {
extra.push("");
extra.push(`OUTPUT BUDGET CLAMP —${lastClamp !== undefined ? ` last: max_tokens ${lastClamp.raw} -> ${lastClamp.capped} (window ${lastClamp.window}, input~${Math.round(lastClamp.planningInput)}) @ ${new Date(lastClamp.at).toISOString()}` : ""} · planning floor ${ledgerFloor} tok from the refused dsh compaction replay (#2490, sticky for the session lifetime). Only compression can recover the input; this clamp guarantees native-window fit.`);
}
// #1097: the processTurn above already resolved the envelope when armed
// (contentStoreOf is idempotent); when disarmed skip the disk read.
const ccrArmed = ccrEnabled(ctx.session);
Expand Down
4 changes: 4 additions & 0 deletions src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1336,6 +1336,10 @@ export function settleUsageReport(
// compaction boundaries via resetSessionCompression (session.ts).
session.stats.lastUsageGradeTokens = session.stats.lastInputTokens;
session.stats.lastInputTokensSource = "usage";
// #3148: a real usage report supersedes the failed-round arm — drop the
// visibility marker so acp_status stops advertising a stale emergency state.
delete session.stats.emergencyShrinkAt;
delete session.stats.emergencyShrinkEstimate;
session.stats.contextTokens = session.stats.lastInputTokens;
session.stats.contextTokensSource = "usage";
if (session.metadata) session.metadata.contextTokensAt = Date.now();
Expand Down
9 changes: 7 additions & 2 deletions src/server/budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -336,10 +336,14 @@ export function clampOutgoingOutput(
* planning basis only ever tightens the cap (fail-closed direction). */
ledgerFloorTokens?: number;
/** #2122/#1933 F1: per-route estimator calibration — scales the local-est
* side of estimateInputTokens so the clamp judges the payload on the same
* provider-billed scale as preflight/nudge. Absent → legacy raw behavior. */ kFactor?: number; kOrigin?: string; origin?: string },
* side of estimateInputTokens so the clamp judges the payload on the same
* provider-billed scale as preflight/nudge. Absent → legacy raw behavior. */ kFactor?: number; kOrigin?: string; origin?: string },
sessionId: string,
log: (level: string, msg: string) => void,
/** #3148: optional visibility hook — invoked with the exact clamp event when
* a cap is applied (callers record it on session.stats for acp_status).
* Never affects the wire decision. */
onClamped?: (info: { raw: number; capped: number; window: number; planningInput: number; ledgerFloor: number }) => void,
): void {
const raw = readOutputBudget(rebuilt, field);
if (typeof raw !== "number") return;
Expand All @@ -365,5 +369,6 @@ export function clampOutgoingOutput(
// rescue right before the turn dies.
const overTarget = ctx.headroomWindow !== undefined && ctx.headroomWindow < ctx.nativeWindow && planningInput >= ctx.headroomWindow;
log("info", `[${sessionId}] output budget clamped ${raw} -> ${capped} (input~${inputEstimate}${ledgerFloor > inputEstimate ? `, planning floor ${ledgerFloor} from the refused dsh compaction replay (#2490)` : ""}, window=${ctx.nativeWindow}); prevents input+output overflow (#453)${overTarget ? `; input already exceeds the headroom-adjusted target ~${ctx.headroomWindow} that nudge/preflight enforce — only compression can recover it, this clamp guarantees native-window fit only (#2096)` : ""}`);
onClamped?.({ raw, capped, window: ctx.nativeWindow, planningInput, ledgerFloor });
}
}
2 changes: 1 addition & 1 deletion src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ export async function prepareGoogle(
}

const rebuilt: GoogleRequestBody = { ...parsed, contents: rebuiltContents, tools: toolsOut, systemInstruction };
clampOutgoingOutput(rebuilt as Record<string, unknown>, "generationConfig.maxOutputTokens", { systemText: googleClientSystem, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "google", rebuilt, opts, upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log);
clampOutgoingOutput(rebuilt as Record<string, unknown>, "generationConfig.maxOutputTokens", { systemText: googleClientSystem, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "google", rebuilt, opts, upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; });
// #532: title-gen side requests carry their own tiny system — skip them.
if (!isTitleGen && googleOutboundSystem !== undefined) {
session.metadata.systemPromptTokens = countSystemAndToolsTokens(googleOutboundSystem, toolsOut);
Expand Down
2 changes: 1 addition & 1 deletion src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ export async function prepareOpenai(
rebuiltMessages = normalizeStrictEchoReasoning(rebuiltMessages, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed)), log, sessionId);
const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined };
warnReasoningPairs(rebuiltMessages, log, sessionId);
clampOutgoingOutput(rebuilt as Record<string, unknown>, typeof (parsed as Record<string, unknown>).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log);
clampOutgoingOutput(rebuilt as Record<string, unknown>, typeof (parsed as Record<string, unknown>).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; });
// prompt_cache_retention is an OpenAI-host-only cache directive; the dsh
// launcher forces PI_CACHE_RETENTION=long (for the session-id
// prompt_cache_key) which makes the client also emit it. Third-party
Expand Down
2 changes: 1 addition & 1 deletion src/server/prepare-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,7 @@ export async function prepareResponses(
const rebuilt: ResponsesRequestBody = { ...parsed, input: rebuiltInput, tools: toolsOut };
warnResponsesReasoningPairs(Array.isArray(rebuiltInput) ? rebuiltInput : [], log, sessionId);
if (!isCompactionTrigger) {
clampOutgoingOutput(rebuilt as Record<string, unknown>, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log);
clampOutgoingOutput(rebuilt as Record<string, unknown>, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; });
}
// Route with the upstream THIS request goes to — session.meta.upstreamOrigin
// is first-wins and would keep injecting pck toward a relay we switched
Expand Down
24 changes: 22 additions & 2 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ import { rewriteResponsesJsonResponseAsync } from "../stream-responses.js";
import { observeResponsesTerminalState } from "../stream-terminal.js";
import { rewriteJsonResponseAsync, type RewriteCtx } from "../stream.js";
import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe.js";
import { clearUpstreamAlertsForHost, recordUpstreamAlert } from "../upstream-alerts.js";
import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason } from "../upstream-alerts.js";
import { formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js";
import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js";
import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering.js";
Expand Down Expand Up @@ -98,6 +98,11 @@ function armFailureShrink(prepared: Prepared, log: (level: string, msg: string)
if (est > s.stats.lastInputTokens) {
s.stats.lastInputTokens = est;
s.stats.lastInputTokensSource = "estimate";
// #3148: the armed state used to be visible only in this warn line —
// record it on the session so acp_status can show it to the host/model.
// Cleared by the next real usage report (settleUsageReport).
s.stats.emergencyShrinkAt = Date.now();
s.stats.emergencyShrinkEstimate = est;
// The error path returns before forward()'s trailing markDirty — the
// arm must schedule its OWN save or it is lost on restart.
markDirty(s);
Expand Down Expand Up @@ -498,6 +503,18 @@ export async function forward(
// instead of blaming the rebuild.
noteInboundMessages(prepared.session, prepared.originalMessages);
}
// #3148: round-level breaker — while the endpoint is provably dead, fail
// fast WITHOUT contacting it instead of burning another full cold send +
// the whole in-request retry ladder per host-driven retry round. The
// thrown detail carries the actionable state (last failure + next probe)
// that the dispatch catch renders into the 502 body.
const circuitBlock = upstreamCircuitBlockReason(upstreamUrl);
if (circuitBlock !== undefined) {
log("warn", `[${prepared?.session.id ?? "passthrough"}] [acp-proxy] upstream circuit open — failing fast without contacting ${upstreamUrl}: ${circuitBlock}`);
if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "upstream circuit open", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate);
if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream circuit open");
throw new Error(`upstream circuit open: ${circuitBlock}`);
}
let upstreamResult: Awaited<ReturnType<typeof fetchWithTimeout>>;
try {
upstreamResult = await fetchWithTransportRetry(upstreamUrl, init, undefined, clientAbort.signal, (info) => {
Expand All @@ -506,16 +523,19 @@ export async function forward(
recordUpstreamConnection(upstreamUrl, proxyUrl);
// #1682: any resolved response proves the host is reachable again.
clearUpstreamAlertsForHost(upstreamUrl);
recordUpstreamCircuitSuccess(upstreamUrl);
} catch (error) {
recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy);
recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined, skippedProxy);
const failureDetail = formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy);
recordUpstreamCircuitFailure(upstreamUrl, error, proxyUrl !== undefined, failureDetail);
// #604: a network-level failure (socket reset, timeout abort) also never
// reports usage — arm the emergency shrink like the 5xx branch below.
if (prepared && req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, log, "network failure", outboundPayloadBreakdown(prepared, opts, route, req.url ?? "").armEstimate);
// [#1343] no response means the attached full text never reached the model —
// drop-and-log it (a corrective note surfaces on the next qualifying request).
if (prepared && prepared.attachedRetrievals && prepared.attachedRetrievals.length > 0) dropRetrievals(prepared.session, prepared.attachedRetrievals.map((i) => i.ref), "upstream network failure");
throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl, skippedProxy)}`, { cause: error });
throw new Error(`upstream request failed: ${failureDetail}`, { cause: error });
}
// #552 learn-on-failure: a converting upstream that rejects a role (codex
// ≥0.153 sends "developer"; vLLM/SGLang-style backends answer 400
Expand Down
10 changes: 10 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,16 @@ export type Session = {
* consumers (upward window self-heal, #496 overflow-evidence gate,
* stale-limit retraction) treat absent as untrusted. */
lastInputTokensSource?: "usage" | "estimate" | "overflow-arm";
/** #3148: emergency shrink armed by a failed round with no usage report
* (armFailureShrink) — when + the armed local estimate. Visibility for
* the state that used to live only in the warn line; acp_status renders
* it. Cleared by the next real usage report (settleUsageReport). */
emergencyShrinkAt?: number;
emergencyShrinkEstimate?: number;
/** #3148: most recent outgoing output-budget clamp event — the #2490
* planning-floor clamps were previously visible only in log lines.
* Display-only: no decision path reads it. */
lastOutputClamp?: { raw: number; capped: number; window: number; planningInput: number; ledgerFloor: number; at: number };
/** #1110: one-shot emergency ceiling armed by an upstream context-
* overflow 400 (server.ts overflow handler) — live evidence this
* session cannot exceed that size. Kept SEPARATE from lastInputTokens
Expand Down
Loading
Loading