Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 46 additions & 2 deletions src/server/budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,53 @@ import { estimateCoreMessages } from "../preflight.js";
import { SELF_HEAL_SUPPRESS_CAP_MS, reductionKeyOf } from "../session-self-heal.js";
import type { Session } from "../session.js";
import { applyEstimateCalibration } from "../util.js";
import { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField } from "./side-request.js";
// #6752: the output-budget field plumbing moved here from side-request.ts —
// it is generic wire plumbing (used by the clamp, the restore path, and the
// watermark fastpath), not side-lane logic, and the prepare modules must stay
// decoupled from the side-lane engine (#1440 P2 cut 2). side-request.ts
// re-exports these, so its existing importers are untouched.
export type OutputBudgetField = "max_tokens" | "max_completion_tokens" | "max_output_tokens" | "generationConfig.maxOutputTokens";

export { outputBudgetField, readOutputBudget, writeOutputBudget };
/** The declared output budget, proto-agnostically. Gemini nests it under
* `generationConfig` (the dotted field name above), the OpenAI/Anthropic
* families keep it flat, so every reader goes through these two accessors. */
export function outputBudgetField(parsed: unknown): OutputBudgetField | null {
if (!parsed || typeof parsed !== "object") return null;
const p = parsed as Record<string, unknown>;
for (const field of ["max_tokens", "max_completion_tokens", "max_output_tokens"] as const) {
if (typeof p[field] === "number" && (p[field] as number) > 0) return field;
}
const gen = p.generationConfig;
if (gen && typeof gen === "object") {
const v = (gen as Record<string, unknown>).maxOutputTokens;
if (typeof v === "number" && v > 0) return "generationConfig.maxOutputTokens";
}
return null;
}

export function readOutputBudget(parsed: Record<string, unknown>, field: OutputBudgetField): number | undefined {
if (field !== "generationConfig.maxOutputTokens") {
const v = parsed[field];
return typeof v === "number" ? v : undefined;
}
const gen = parsed.generationConfig;
if (!gen || typeof gen !== "object") return undefined;
const v = (gen as Record<string, unknown>).maxOutputTokens;
return typeof v === "number" ? v : undefined;
}

export function writeOutputBudget(parsed: Record<string, unknown>, field: OutputBudgetField, value: number): void {
// #2011: output-budget fields are integer-typed on every wire protocol. Coerce at this
// sole writer so NO caller can emit a fractional token count — the clamp path already
// floors, but this also guards the restore path and any future caller at the type boundary.
const tokens = Number.isInteger(value) ? value : Math.floor(value);
if (field !== "generationConfig.maxOutputTokens") {
parsed[field] = tokens;
return;
}
const gen = parsed.generationConfig;
parsed.generationConfig = { ...(gen && typeof gen === "object" ? (gen as Record<string, unknown>) : {}), maxOutputTokens: tokens };
}

// #453 hard backstop: cap the forwarded output budget so input+output can never
// exceed the window on request-rebuilding upstreams (vLLM rejects an oversized
Expand Down
15 changes: 14 additions & 1 deletion src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import { reconcileSystemAnchor } from "../system-anchor.js";
import { logRawStripCounts, markerArtifactCount, stripRawStage, type RawStripCounts } from "../marker/engine.js";
import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js";
import { applyImageRescalePass } from "../image-rescale.js";
import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js";
import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, outputBudgetField, readOutputBudget, writeOutputBudget } from "./budget.js";
import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js";
import { injectOpenaiTool, injectTool } from "./inject.js";
import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js";
Expand Down Expand Up @@ -139,6 +139,16 @@ export async function prepareOpenai(
log("debug", `[${session.id}] fastpath ${wpOutcome.hit ? "HIT" : wpOutcome.decision}${wpOutcome.creditedMiss ? "(credited)" : ""}${wpOutcome.measuredT !== undefined ? ` T=${wpOutcome.measuredT}` : ""}`);
if (wpOutcome.hit) {
wpHit = wpOutcome.hit;
// #6752: handle.ts already ran restoreOutputBudget on `parsed` —
// the rebind below re-parses scalar fields from the raw wire text,
// which would silently discard the restored budget (the log claims
// the restore while the wire still carries the client's shrunk
// value). Carry the restored budget across the rebind.
const budgetField = outputBudgetField(parsed);
const restoredBudget =
budgetField !== null
? readOutputBudget(parsed as Record<string, unknown>, budgetField)
: undefined;
const fields: Record<string, unknown> = {};
for (const [key, span] of wpHit.fieldSpans) {
// "messages" keeps its WIRE POSITION (the scanner records it
Expand All @@ -147,6 +157,9 @@ export async function prepareOpenai(
if (key === "messages") fields[key] = wpHit.windowMessages;
else fields[key] = JSON.parse(bodyText.slice(span.start, span.end));
}
if (budgetField !== null && restoredBudget !== undefined) {
writeOutputBudget(fields, budgetField, restoredBudget);
}
parsed = fields as OpenAIRequestBody;
inboundMessageCount = wpHit.totalT;
}
Expand Down
11 changes: 9 additions & 2 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,13 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
// bytes. Protocol-neutral (any JSON-object body), unlike the role
// rewrite above which is openai/responses-only.
compatDropPaths = resolveCompatDropFields(opts.routes, upstreamUrl, opts.compat?.dropFields);
// #6752: when the drop overrides an earlier "output budget restored" log,
// say so explicitly so the log is honest (the restore fired, but the field
// is gone by the time the wire is built).
const OUTPUT_BUDGET_DROP_PATHS = ["max_tokens", "max_completion_tokens", "max_output_tokens", "generationConfig.maxOutputTokens"];
const budgetOverrideNote = compatDropPaths.some(p => OUTPUT_BUDGET_DROP_PATHS.includes(p))
? " (overrides the earlier 'output budget restored' log — the field is absent from the forwarded body)"
: "";
if (compatDropPaths.length > 0 && typeof wireBody === "string") {
// #564/#2627: drops run on the SAME graph as every other transform, so
// identity decides — not the drop path's shape. A drop that deletes or
Expand All @@ -697,14 +704,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
const dropped = dropCompatFieldsJson(graph, compatDropPaths);
if (dropped > 0) {
commitGraph();
log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")}) (#1757)`);
log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")})${budgetOverrideNote} (#1757)`);
}
} else {
const applied = applyCompatDropFields(wireBody, compatDropPaths);
if (applied.dropped > 0) {
wireBody = applied.body;
invalidateSeparatorOwnership();
log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${applied.dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")}) (#1757)`);
log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${applied.dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")})${budgetOverrideNote} (#1757)`);
}
}
}
Expand Down
49 changes: 6 additions & 43 deletions src/server/side-request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ import { type ResolvedImageBilling } from "../image-tokens.js";
import { ACP_TOOL_NAMES, ABSORB_TOOL_NAME, BLOCK_OP_TOOL_NAMES, RETRIEVE_TOOL_NAME, RULE_TOOL_NAME } from "acp-kernel";
import { READ_ORIGINAL_IMAGE_TOOL_NAME } from "../compress-tool.js";
import { dshCarrierRawMarkers } from "../durable-message-guards.js";
import { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField } from "./budget.js";

// #6752: the output-budget field plumbing lives in budget.ts now (generic wire
// plumbing, not side-lane logic); re-exported here so existing importers
// (server.ts, tests) keep working unchanged.
export { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField };

// #388: side requests (title-gen etc.) share the main session key but must not
// touch kernel state. Identified by a tiny output budget (same heuristic as
Expand Down Expand Up @@ -607,49 +613,6 @@ export function stripLeakedBiliTools(parsed: unknown): boolean {
return true;
}

export type OutputBudgetField = "max_tokens" | "max_completion_tokens" | "max_output_tokens" | "generationConfig.maxOutputTokens";

/** The declared output budget, proto-agnostically. Gemini nests it under
* `generationConfig` (the dotted field name above), the OpenAI/Anthropic
* families keep it flat, so every reader goes through these two accessors. */
export function outputBudgetField(parsed: unknown): OutputBudgetField | null {
if (!parsed || typeof parsed !== "object") return null;
const p = parsed as Record<string, unknown>;
for (const field of ["max_tokens", "max_completion_tokens", "max_output_tokens"] as const) {
if (typeof p[field] === "number" && (p[field] as number) > 0) return field;
}
const gen = p.generationConfig;
if (gen && typeof gen === "object") {
const v = (gen as Record<string, unknown>).maxOutputTokens;
if (typeof v === "number" && v > 0) return "generationConfig.maxOutputTokens";
}
return null;
}

export function readOutputBudget(parsed: Record<string, unknown>, field: OutputBudgetField): number | undefined {
if (field !== "generationConfig.maxOutputTokens") {
const v = parsed[field];
return typeof v === "number" ? v : undefined;
}
const gen = parsed.generationConfig;
if (!gen || typeof gen !== "object") return undefined;
const v = (gen as Record<string, unknown>).maxOutputTokens;
return typeof v === "number" ? v : undefined;
}

export function writeOutputBudget(parsed: Record<string, unknown>, field: OutputBudgetField, value: number): void {
// #2011: output-budget fields are integer-typed on every wire protocol. Coerce at this
// sole writer so NO caller can emit a fractional token count — the clamp path already
// floors, but this also guards the restore path and any future caller at the type boundary.
const tokens = Number.isInteger(value) ? value : Math.floor(value);
if (field !== "generationConfig.maxOutputTokens") {
parsed[field] = tokens;
return;
}
const gen = parsed.generationConfig;
parsed.generationConfig = { ...(gen && typeof gen === "object" ? (gen as Record<string, unknown>) : {}), maxOutputTokens: tokens };
}

// #3071: absolute capability floor for the output budget when NO source knows a
// ceiling for the model. The client's raw-history-derived budget decays unbounded
// (…, 3071, 2061, 252, <=200); "last non-starved wins" then pins the session at a
Expand Down
Loading
Loading