Skip to content
5 changes: 4 additions & 1 deletion src/image-tokens.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,10 @@ export function pixelTileEstimate(w: number, h: number): number {
return 85 + 170 * tiles;
}

function base64ImageCost(b64: string, billing: ResolvedImageBilling, configuredCap?: number): number {
/** Cap-applied token cost of one base64 image under `billing`. Exported so the
* CoreMessage-level reserve (server.ts outboundImageReserve) charges pixels on
* exactly the same scale the wire-body estimators use. */
export function base64ImageCost(b64: string, billing: ResolvedImageBilling, configuredCap?: number): number {
if (billing === "pixels") {
const dims = decodeImageDims(b64);
return applyCap(dims ? pixelTileEstimate(dims.w, dims.h) : PIXEL_IMAGE_FALLBACK_TOKENS, configuredCap);
Expand Down
46 changes: 41 additions & 5 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import net from "node:net";
import tls from "node:tls";
import { createHash, randomUUID } from "node:crypto";
import { performance } from "node:perf_hooks";
import { createCore, type CompressionCore, type CompressionState, type Config, type CoreMessage, type NudgeDecision, type Prompts, type PackSurface, defaultPrompts, deactivateBlock } from "acp-kernel";
import { createCore, prune, type CompressionCore, type CompressionState, type Config, type CoreMessage, type NudgeDecision, type Prompts, type PackSurface, defaultPrompts, deactivateBlock } from "acp-kernel";
import { resolveCompress, resolveCompressPrompts, resolveCompressSurfaceDetailed, resolveRequestConfig, type ResolvedKernelConfig } from "./compress-settings.js";
import { dropCompressReasoningIds, type CompressReasoningConfig } from "./reasoning-drop.js";
import type { ProxyOptions, ResignSettings } from "./config.js";
Expand Down Expand Up @@ -76,7 +76,7 @@ import {
import { ABSORB_TOOL_NAME, COMPRESS_TOOL, BILI_ACP_TOOLS_ANTHROPIC, BILI_ACP_TOOLS_ANTHROPIC_NO_RANGE, BILI_ACP_TOOLS_GOOGLE, BILI_ACP_TOOLS_GOOGLE_NO_RANGE, BILI_ACP_TOOLS_OPENAI, BILI_ACP_TOOLS_OPENAI_NO_RANGE, BILI_ACP_TOOLS_RESPONSES, BILI_ACP_TOOLS_RESPONSES_NO_RANGE, BILI_ACP_READONLY_TOOLS_RESPONSES, BILI_ACP_READONLY_TOOLS_RESPONSES_NO_RANGE, COMPRESS_TOOL_NAME, RULE_TOOL, RULE_TOOL_GOOGLE, RULE_TOOL_OPENAI, RULE_TOOL_RESPONSES, absorbToolsFor, retrieveToolsFor, buildAcpTagsOnlyPrompt } from "./compress-tool.js";
import { absorbEnabled, storeEffectiveAbsorb } from "./absorb.js";
import { ccrEnabled, ccrLoopConfig, ccrPluginWireOk, contentStoreOf, executeRetrieve, pruneExpiredRetrievals, reconcileReloadedRetrievals, renderRetrievalNotes, retrieveToolName, snapshotPendingRetrievals, snapshotRetrievalNotes, storeEffectiveCcr, type CcrSettings } from "./store.js";
import { buildIncomingImageIndex, pruneRetrieveImgExports } from "./image-restore.js";
import { buildIncomingImageIndex, messageImageBytes, pruneRetrieveImgExports } from "./image-restore.js";
import { rulesEnabled, storeEffectiveRules } from "./rules-feature.js";
import { storeEffectiveSearchPlanAware } from "./decompress-shared.js";
import { armAutoFoldBackoff, autoFoldEngaged, AUTO_FOLD_BACKOFF_MS, externalQueueDroppedAll, withExternalSummaryTools } from "./external-summary-surface.js";
Expand All @@ -88,7 +88,7 @@ import { buildSessionCacheReport, handleAcpCache, learnedImageReserve, noteClien
import { extractBillingAttributionBlock, extractSummaryFromSse, preflightCompress, estimateCoreMessages, estimateCoreMessagesUpper, estimateCoreMessagesUpperBytes, estimateRawBodyTokens, type PreflightResult } from "./preflight.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, DEFAULT_DECIDE_MAX_TOKENS, DECIDE_TIMEOUT_MS, extractDecisionText, ladderMode, parseDecision, recordDecision, resolveDecisionRange, type DecideConfig, type DecisionOutcome } from "./nudge-decide.js";
import { gcConfigFromEnv, gcSessionFiles } from "./session-gc.js";
import { countImagesInParsedBody, countImagesInRawBody, imageTokensInRawBody, imageTokensInParsedBody, resolveImageBilling, upstreamHost, type ResolvedImageBilling } from "./image-tokens.js";
import { base64ImageCost, countImagesInParsedBody, countImagesInRawBody, imageTokensInRawBody, imageTokensInParsedBody, resolveImageBilling, upstreamHost, type ResolvedImageBilling } from "./image-tokens.js";
import { APIG_RESIGN_HEADER, APIG_RESIGN_CREDENTIAL_HEADER, APIG_RESIGN_SCHEME, KNOWN_SIGNATURE_SCHEMES, clearSignedRefusal, decodeApigCredential, inboundSignedScheme, readPendingRefusals, recordSignedRefusal, resignApig, signedRefusal, unresolvedRefusals, type ApigCredential } from "./apig-resign.js";
import { renderUI, handleConfigGet, handleConfigPut, handleSummaryCredentialPut, buildOverview, buildSessionList, buildSessionPage, buildSessionDetail, hiddenEmptyCount } from "./web/index.js";
import { reapOrphanBlocks } from "./orphan-gc.js";
Expand Down Expand Up @@ -1693,6 +1693,42 @@ export function inboundResignCredential(
return decodeApigCredential(encoded);
}

// #4060: the OUTBOUND image reserve — the same image mass imageReserveFor
// charges, but measured on the messages that will actually ride the wire
// instead of on the inbound request body. The client resends its whole history
// every turn, folded images included, so an inbound-body reserve keeps charging
// for pixels the egress prune already dropped: a long screenshot session
// reserves hundreds of images that never leave the proxy, the window meter reads
// several hundred percent, and the nudge emergency-suppresses the very folds
// that would shrink the payload. Learning already measures the OUTBOUND body
// (noteForwardedImageFacts on wireBody), so consuming the inbound one was an
// asymmetry, not a design. Returns 0 when no image survives the prune — the
// honest reserve for a text-only wire. Carrier-shape lockstep: messageImageBytes
// (this reader) and the image-tokens.ts raw-body walkers (learning-side count +
// inbound cost) are separate walkers — a NEW carrier shape must be added to both
// or the reserve silently undercharges it (#3398 discipline).
export function outboundImageReserve(
session: Session,
msgs: CoreMessage[],
opts: ProxyOptions,
upstreamUrl: string | undefined,
): number {
const billing = imageBillingFor(opts, upstreamUrl);
const cap = imageTokenCapFor(opts, upstreamUrl);
let prior = 0;
let nImages = 0;
for (const m of prune(msgs, session.state)) {
for (const img of messageImageBytes(m)) {
if (img.b64.length === 0) continue;
prior += base64ImageCost(img.b64, billing, cap);
nImages += 1;
}
}
if (nImages === 0) return 0;
const learned = learnedImageReserve(session, upstreamHost(upstreamUrl), nImages, `${billing}:${cap}`, cap);
return learned === undefined ? prior : Math.min(learned, prior);
}

// #1843 L1: the IMAGE-channel reserve for a payload — the prior-based estimate
// (pixels/bytes per the resolved billing + cap) upgraded to LEARNED truth when
// this session holds fresh usage-learned evidence for this route (per-image
Expand Down Expand Up @@ -2320,7 +2356,7 @@ export function diagNudge(turn: { nudge?: { shouldInject: boolean; reason: strin
// shrinks → the next estimate drops). Turn 1 of a fresh explicit session
// still feeds 0 (nothing measured yet; nothing pending either), so
// first-turn behavior is byte-identical to pre-#728.
export function effectiveTokenCount(session: Session, msgs: CoreMessage[], inboundImageTokens = 0): { tokens: number; source: "usage" | "estimate" } {
export function effectiveTokenCount(session: Session, msgs: CoreMessage[], imageTokens = 0): { tokens: number; source: "usage" | "estimate" } {
// #1492: only usage-grade baselines are authoritative sizing inputs. An
// estimate-sourced value describes ONE turn's outbound — possibly the FULL
// raw history on an unfolded fallback turn — and pinning the nudge to it
Expand All @@ -2342,7 +2378,7 @@ export function effectiveTokenCount(session: Session, msgs: CoreMessage[], inbou
// reporting upstreams fall back to legacy sizing rather than a frozen meter.
const anchored = postRebuildAnchorTokens(session);
if (anchored > 0) return { tokens: anchored, source: "estimate" };
const raw = estimateCoreMessagesUpper(msgs) + inboundImageTokens;
const raw = estimateCoreMessagesUpper(msgs) + imageTokens;
// #1569/#1839: while the latest baseline is not usage-grade (the transient
// window right after a failed turn), sizing on ANY re-derived view is how
// ghosts enter: #1569 first tried min(est, raw) — the char-count upper
Expand Down
11 changes: 7 additions & 4 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { rewriteInboundThreadContinue } from "./thread-tracking.js";
import { captureThreadsTurn, estimateThreadsTokens, isThreadsBody, threadsReforgeEnabled } from "./threads-lane.js";
import type { CompressReasoningConfig } from "../reasoning-drop.js";
import type { ProxyOptions } from "../config.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, outboundImageReserve, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
Expand Down Expand Up @@ -208,13 +208,16 @@ export async function prepareAnthropic(
// #1320: signature-only thinking blocks bill restored thinking tokens upstream
// but project as empty text locally — attribute the provider-vs-local residual
// to them so every meter sees the billed context (metering-only, wire untouched).
const inboundImageTokens = imageReserveFor(session, "anthropic", parsed, opts, upstreamOrigin);
// #3952: the image mass subtracted from that residual is the OUTBOUND one —
// the residual is provider-minus-local for the payload actually sent, so
// charging images the prune already dropped would swallow the whole gap.
const outboundImageTokens = outboundImageReserve(session, renderMsgs, opts, upstreamOrigin);
projectThinkingMass(msgs, {
providerInputTokens: session.stats.lastInputTokens,
measured: session.stats.lastInputTokensSource === "usage",
systemText: extractSystem(parsed.system),
tools: parsed.tools,
imageTokens: inboundImageTokens,
imageTokens: outboundImageTokens,
storedOverhead: typeof session.metadata.systemPromptTokens === "number" ? session.metadata.systemPromptTokens : undefined,
});
// #1001: pre-turn snapshot — processTurn below assigns fresh refs to every
Expand All @@ -233,7 +236,7 @@ export async function prepareAnthropic(
// zero-baseline forks replay their FULL raw history with no measurement,
// so feeding 0 blinds the nudge (usage 0%, growth ref 0) and no
// compression trigger fires until overflow. See effectiveTokenCount.
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, inboundImageTokens);
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, outboundImageTokens);
const activeBefore = new Set(session.state.blocks.filter((b) => b.active).map((b) => b.blockId));
// Absorb markers are injected by the kernel's processTurn from
// config.absorb. With no channel to call the tool (injection off),
Expand Down
4 changes: 2 additions & 2 deletions src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { renderNudgeText, viableRanges } from "acp-kernel";
import { coreToGoogle, googleToCore, normalizeGoogleUserPartOrder, type BiliMessage, type GoogleContent, type GoogleRequestBody, type GoogleSystemInstruction, type GoogleTool } from "acp-kernel/wire";
import type { ProxyOptions } from "../config.js";
import { currentCalibrationFactor } from "../util.js";
import { diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, usageGradeInputBaseline, type Prepared } from "../server.js";
import { diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, outboundImageReserve, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, usageGradeInputBaseline, type Prepared } from "../server.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
Expand Down Expand Up @@ -135,7 +135,7 @@ export async function prepareGoogle(
// #1001: pre-turn snapshot — processTurn below assigns fresh refs to every
// previously-unknown id, which would make rewrite detection read 1.0.
const knownRefsBefore = new Set(Object.keys(session.state.messageRefs.byRaw));
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, imageReserveFor(session, "google", parsed, opts, upstreamOrigin));
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, outboundImageReserve(session, renderMsgs, opts, upstreamOrigin));
const activeBefore = new Set(session.state.blocks.filter((b) => b.active).map((b) => b.blockId));
const absorbBlock = effectiveAbsorbBlock(pluginMode, config, opts.compress.absorb);
const absorbTools = absorbToolsFor(absorbBlock?.toolName ?? ABSORB_TOOL_NAME);
Expand Down
4 changes: 2 additions & 2 deletions src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import type { CompressReasoningConfig } from "../reasoning-drop.js";
import type { ProxyOptions } from "../config.js";
import { currentCalibrationFactor, hardenOpenaiAssistantContent, systemToUser } from "../util.js";
import { conversationHeaderSource, resolveRelayAffinityPck } from "../session-id.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnDroppedOpenaiParts, warnReasoningPairs, type Prepared } from "../server.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, outboundImageReserve, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnDroppedOpenaiParts, warnReasoningPairs, type Prepared } from "../server.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
Expand Down Expand Up @@ -128,7 +128,7 @@ export async function prepareOpenai(
// tokenCount = upstream's real input_tokens from the previous turn
// tokenCount = upstream's real input_tokens from the previous turn
// (see anthropic branch comment + its #553-follow-up exception).
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, imageReserveFor(session, "openai", parsed, opts, billingUpstream ?? upstreamOrigin));
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, outboundImageReserve(session, renderMsgs, opts, billingUpstream ?? upstreamOrigin));

const activeBefore = new Set(session.state.blocks.filter((b) => b.active).map((b) => b.blockId));
// Absorb markers ride in the kernel's processTurn output (gated by
Expand Down
4 changes: 2 additions & 2 deletions src/server/prepare-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import type { CompressReasoningConfig } from "../reasoning-drop.js";
import { resolveAbsorbSettings } from "../compress-settings.js";
import { resolveCompressProtocol, type CompressSettings, type ProxyOptions } from "../config.js";
import { currentCalibrationFactor, strippedResponseIdWarning } from "../util.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, repairResponsesAssistantOrdering, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, usageGradeInputBaseline, warnResponsesReasoningPairs, type Prepared } from "../server.js";
import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, outboundImageReserve, planReasoningDrop, repairResponsesAssistantOrdering, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, usageGradeInputBaseline, warnResponsesReasoningPairs, type Prepared } from "../server.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { mergeAdjacentConfigurationUpdates, patchResponsesInputWithToolImages as patchResponsesInput, responsesToCoreWithToolImages as responsesToCore } from "../responses-tool-output.js";
Expand Down Expand Up @@ -243,7 +243,7 @@ export async function prepareResponses(
if (opts.debug) {
log("info", `[${sessionId}] input items: ${Array.isArray(parsed.input) ? parsed.input.map((i: ResponseInputItem) => i.type).join(",") : "(string)"}`);
}
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, imageReserveFor(session, "responses", parsed, opts, billingUpstream ?? upstreamOrigin));
const { tokens: tokenCount, source: tokenCountSource } = effectiveTokenCount(session, renderMsgs, outboundImageReserve(session, renderMsgs, opts, billingUpstream ?? upstreamOrigin));
// Absorb markers ride in the kernel's processTurn output (gated by
// config.absorb). The marker/text protocol has no native tool channel,
// so strip absorb from the loop config there (both modes).
Expand Down
Loading
Loading