Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ billion-context/
│ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548)
│ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805)
│ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221)
│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922)
│ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers
│ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062)
│ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346)
Expand Down
7 changes: 2 additions & 5 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ import { prepareResponses, prepareResponsesCompact } from "./server/prepare-resp
import { FORCE_TEXT_PROTOCOL } from "./server/inject.js";
export { isChatGptCodexUpstream, isCodexResponsesLite, resolvePromptCacheKey, shouldInjectPromptCacheKey } from "./server/prepare-responses.js";
import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, nudgeSuppressed, pluginLaneDegraded, pluginLaneRestore } from "./session-self-heal.js";
import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, type NudgeStamps, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, splitSessionWarnings, usageAnchorCycleStale, staleSmallAnchorFallback } from "./session.js";
import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, type NudgeStamps, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, applyPostRebuildBaselineAnchor, splitSessionWarnings, usageAnchorCycleStale, staleSmallAnchorFallback } from "./session.js";
import type { SeparatorManifest } from "./server/responses-ordering.js";
import { detectStaleInstall, isVersionNewer } from "./update.js";
import { decideEaddrinuse, eaddrinuseExitCode, isPortBusyError, postBindCanLadder, UPGRADE_REPLACEMENT_ENV, type FailFastReason } from "./listen-policy.js";
Expand Down Expand Up @@ -4374,10 +4374,7 @@ export async function preflightCompressIfNeeded(
const rebuiltImageTokens = imageReserveFor(session, rebuilt.protocol, rebuilt.body, opts, upstreamUrl);
remainingImageTokens = rebuiltImageTokens;
reducedImageWire = result.compressedRanges > 0 && rebuiltImageTokens < imageTokens;
if (rebuiltTextSize > session.stats.lastInputTokens) {
session.stats.lastInputTokens = rebuiltTextSize;
session.stats.lastInputTokensSource = "estimate";
}
applyPostRebuildBaselineAnchor(session, rebuiltTextSize);
// Same calibrated caliber as the trigger above — gate, per-round exit
// and this final fit must judge the payload on one scale (#1933 F1).
// The baseline anchor above stays raw deliberately: it is a floor for
Expand Down
19 changes: 10 additions & 9 deletions src/server/budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,16 @@ export { outputBudgetField, readOutputBudget, writeOutputBudget };

// #453 hard backstop: cap the forwarded output budget so input+output can never
// exceed the window on request-rebuilding upstreams (vLLM rejects an oversized
// total instead of clamping). Wired into the openai/responses/google prepare
// lanes only — canonical Anthropic routes enforce their input limit
// independently of max_tokens, so they get no clamp call site. #4205 boundary:
// wrapper providers on the anthropic wire (completion billed against the same
// window) DO reserve headroom at the band layer but keep no egress clamp here;
// their residual over-requests ride the overflow self-heal (#4093) instead.
// The proxy owns both sides of the sum, so capping output to (window - input -
// margin) makes the overflow impossible even when the agent ignores the
// compress nudge.
// total instead of clamping). Wired into ALL FOUR prepare lanes — openai,
// responses, google, and anthropic (#5880 added the anthropic site: wrapper
// providers on that wire bill completion against the SAME stated window as
// input — DeepSeek's 400 quotes "N in the messages, M in the completion"
// against one wall — which is exactly the shape this clamp exists for; for
// canonical Anthropic, whose input limit is enforced independently of
// max_tokens, the requested budget already rides under the window and the
// clamp is a no-op). The proxy owns both sides of the sum, so capping output
// to (window - input - margin) makes the overflow impossible even when the
// agent ignores the compress nudge.
const OUTPUT_CLAMP_MARGIN_PCT = 0.05;
const OUTPUT_CLAMP_MIN_MARGIN = 2048;
// #5278: safety inflation applied to the planning estimate BEFORE sizing both
Expand Down
2 changes: 1 addition & 1 deletion src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3132,7 +3132,7 @@ async function handleRequest(
return await prepareGoogle(work as GoogleRequestBody, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, nativeWindow, googleModel ?? v1Model, googlePathKind(urlPath) === "stream-generate", visibilityMarkers, upstreamOrigin, req, decide);
}
const preparedOut = protocol === "anthropic"
? await prepareAnthropic(work as AnthropicRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, reasoningCfg, visibilityMarkers, requestModel, decide)
? await prepareAnthropic(work as AnthropicRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, nativeWindow, reasoningCfg, visibilityMarkers, requestModel, decide)
: protocol === "openai"
? await prepareOpenai(work as OpenAIRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, nativeWindow, reasoningCfg, visibilityMarkers, route?.rewrittenUrl, decide, bodyText)
: responsesCompact
Expand Down
14 changes: 12 additions & 2 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import { captureThreadsTurn, estimateThreadsTokens, isThreadsBody, threadsReforg
import type { CompressReasoningConfig } from "../reasoning-drop.js";
import type { ProxyOptions } from "../config.js";
import { pluginAgentHeader } from "../plugin.js";
import { applyReasoningDrop, dedupeAdjacentOutputConfig, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js";
import { applyReasoningDrop, dedupeAdjacentOutputConfig, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js";
import { currentCalibrationFactor } from "../util.js";
import { recordNudgeObservation } from "../context-analysis.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { emitNudgeInjected, nudgeDebugMeta } from "../nudge-debug.js";
Expand Down Expand Up @@ -40,7 +41,7 @@ import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker
import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js";
import { applyImageRescalePass } from "../image-rescale.js";
import { isAutoModeClassifier } from "../auto-mode-forge.js";
import { countSystemAndToolsTokens, emergencyNudge, emergencyNudgeArmed, projectThinkingMass } from "./budget.js";
import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, projectThinkingMass } from "./budget.js";
import { effectiveAbsorbBlock } from "./prepare-responses.js";
import { effectiveBlockOpsEnabled } from "./prepare-responses.js";
import { injectSystem, injectTool } from "./inject.js";
Expand All @@ -63,6 +64,7 @@ export async function prepareAnthropic(
session: Session,
pluginMode: boolean,
upstreamOrigin: string,
nativeWindow: number,
reasoning: CompressReasoningConfig | undefined,
visibilityMarkers: boolean,
/** The route's resolved model, when the caller already has one. On the
Expand Down Expand Up @@ -693,6 +695,14 @@ export async function prepareAnthropic(
// main-scale tiebreak sees this request's own upper bound.
probeRequestMembership(session, collectRequestIds(incomingMsgIds, originalMessages), session.stats.localInputEstimate ?? 0);
if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin;
// #5880: egress output-budget clamp, same call-site family as the openai/
// responses/google lanes (prepare-openai.ts). Wrapper providers on this
// wire bill completion against the SAME stated window as input (DeepSeek's
// 400: "N in the messages, M in the completion" against one wall), so an
// input-only fit judgment must be backed by an output clamp here too; for
// canonical Anthropic (separate input/output limits) the requested
// max_tokens already rides under the window and the clamp is a no-op.
clampOutgoingOutput(rebuilt as Record<string, unknown>, "max_tokens", { systemText: extractSystem(systemOut), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserve, kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record<string, unknown>, opts.routes, upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; });
if (nudgeOutcome === "injected") recordNudgeInject(session, nudge);
return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, attachedRetrievals, attachedRetrievalNoteIds, processedMessages, originalMessages, judgeBaseline, anthropicSystem: parsed.system, anthropicBillingBlock, anthropicCacheMarks, anthropicClientCacheControls: clientCacheControls, systemNotes: sysNotes, protocol: "anthropic", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, nudgeStampsBefore: nudgeOutcome === "injected" ? nudgeStampsBefore : undefined, prompts, surface, renderTags: renderStrategy, dropReasoning: (msgs: BiliMessage[]) => applyReasoningDrop(msgs, reasoningPlan, log, sessionId) } as Prepared;
}
Expand Down
16 changes: 16 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1192,6 +1192,22 @@ export function setPostRebuildAnchor(session: Session, tokens: number): void {
markDirty(session);
}

// #1987 post-rebuild baseline anchor, hardened by #5880: an upstream-proven
// overflow-arm tag must NEVER be demoted to estimate-grade by a local payload
// estimate — demoting drops the evidence floor that keeps an armed session
// folding instead of trusting optimistic estimates (same doctrine as
// armFailureShrink, relay.ts #2484). In the #5880 incident the caliber delta
// between the netted arm value and a fresh estimate of the rebuilt view tipped
// the inequality on every fold turn, silently disarming the session so its
// estimate-only forwards 400'd turn after turn.
export function applyPostRebuildBaselineAnchor(session: Session, rebuiltTextSize: number): void {
if (session.stats.lastInputTokensSource === "overflow-arm") return;
if (rebuiltTextSize > session.stats.lastInputTokens) {
session.stats.lastInputTokens = rebuiltTextSize;
session.stats.lastInputTokensSource = "estimate";
}
}

/** Metadata is persisted user-editable JSON — re-validate on read; a corrupt
* stamp degrades to "no anchor" (legacy sizing) instead of poisoning the meter. */
export function postRebuildAnchorTokens(session: Session | undefined): number {
Expand Down
Loading
Loading