diff --git a/src/server/budget.ts b/src/server/budget.ts index fe1fa22c22..ccac831866 100644 --- a/src/server/budget.ts +++ b/src/server/budget.ts @@ -4,9 +4,53 @@ import { estimateCoreMessages } from "../preflight.js"; import { SELF_HEAL_SUPPRESS_CAP_MS, reductionKeyOf } from "../session-self-heal.js"; import type { Session } from "../session.js"; import { applyEstimateCalibration } from "../util.js"; -import { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField } from "./side-request.js"; +// #6752: the output-budget field plumbing moved here from side-request.ts — +// it is generic wire plumbing (used by the clamp, the restore path, and the +// watermark fastpath), not side-lane logic, and the prepare modules must stay +// decoupled from the side-lane engine (#1440 P2 cut 2). side-request.ts +// re-exports these, so its existing importers are untouched. +export type OutputBudgetField = "max_tokens" | "max_completion_tokens" | "max_output_tokens" | "generationConfig.maxOutputTokens"; -export { outputBudgetField, readOutputBudget, writeOutputBudget }; +/** The declared output budget, proto-agnostically. Gemini nests it under + * `generationConfig` (the dotted field name above), the OpenAI/Anthropic + * families keep it flat, so every reader goes through these two accessors. */ +export function outputBudgetField(parsed: unknown): OutputBudgetField | null { + if (!parsed || typeof parsed !== "object") return null; + const p = parsed as Record; + for (const field of ["max_tokens", "max_completion_tokens", "max_output_tokens"] as const) { + if (typeof p[field] === "number" && (p[field] as number) > 0) return field; + } + const gen = p.generationConfig; + if (gen && typeof gen === "object") { + const v = (gen as Record).maxOutputTokens; + if (typeof v === "number" && v > 0) return "generationConfig.maxOutputTokens"; + } + return null; +} + +export function readOutputBudget(parsed: Record, field: OutputBudgetField): number | undefined { + if (field !== "generationConfig.maxOutputTokens") { + const v = parsed[field]; + return typeof v === "number" ? v : undefined; + } + const gen = parsed.generationConfig; + if (!gen || typeof gen !== "object") return undefined; + const v = (gen as Record).maxOutputTokens; + return typeof v === "number" ? v : undefined; +} + +export function writeOutputBudget(parsed: Record, field: OutputBudgetField, value: number): void { + // #2011: output-budget fields are integer-typed on every wire protocol. Coerce at this + // sole writer so NO caller can emit a fractional token count — the clamp path already + // floors, but this also guards the restore path and any future caller at the type boundary. + const tokens = Number.isInteger(value) ? value : Math.floor(value); + if (field !== "generationConfig.maxOutputTokens") { + parsed[field] = tokens; + return; + } + const gen = parsed.generationConfig; + parsed.generationConfig = { ...(gen && typeof gen === "object" ? (gen as Record) : {}), maxOutputTokens: tokens }; +} // #453 hard backstop: cap the forwarded output budget so input+output can never // exceed the window on request-rebuilding upstreams (vLLM rejects an oversized diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 59d9ac4fae..f875e0771c 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -41,7 +41,7 @@ import { reconcileSystemAnchor } from "../system-anchor.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage, type RawStripCounts } from "../marker/engine.js"; import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; import { applyImageRescalePass } from "../image-rescale.js"; -import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js"; +import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, outputBudgetField, readOutputBudget, writeOutputBudget } from "./budget.js"; import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js"; import { injectOpenaiTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; @@ -139,6 +139,16 @@ export async function prepareOpenai( log("debug", `[${session.id}] fastpath ${wpOutcome.hit ? "HIT" : wpOutcome.decision}${wpOutcome.creditedMiss ? "(credited)" : ""}${wpOutcome.measuredT !== undefined ? ` T=${wpOutcome.measuredT}` : ""}`); if (wpOutcome.hit) { wpHit = wpOutcome.hit; + // #6752: handle.ts already ran restoreOutputBudget on `parsed` — + // the rebind below re-parses scalar fields from the raw wire text, + // which would silently discard the restored budget (the log claims + // the restore while the wire still carries the client's shrunk + // value). Carry the restored budget across the rebind. + const budgetField = outputBudgetField(parsed); + const restoredBudget = + budgetField !== null + ? readOutputBudget(parsed as Record, budgetField) + : undefined; const fields: Record = {}; for (const [key, span] of wpHit.fieldSpans) { // "messages" keeps its WIRE POSITION (the scanner records it @@ -147,6 +157,9 @@ export async function prepareOpenai( if (key === "messages") fields[key] = wpHit.windowMessages; else fields[key] = JSON.parse(bodyText.slice(span.start, span.end)); } + if (budgetField !== null && restoredBudget !== undefined) { + writeOutputBudget(fields, budgetField, restoredBudget); + } parsed = fields as OpenAIRequestBody; inboundMessageCount = wpHit.totalT; } diff --git a/src/server/relay.ts b/src/server/relay.ts index a41da78297..0355a8d3e7 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -686,6 +686,13 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // bytes. Protocol-neutral (any JSON-object body), unlike the role // rewrite above which is openai/responses-only. compatDropPaths = resolveCompatDropFields(opts.routes, upstreamUrl, opts.compat?.dropFields); + // #6752: when the drop overrides an earlier "output budget restored" log, + // say so explicitly so the log is honest (the restore fired, but the field + // is gone by the time the wire is built). + const OUTPUT_BUDGET_DROP_PATHS = ["max_tokens", "max_completion_tokens", "max_output_tokens", "generationConfig.maxOutputTokens"]; + const budgetOverrideNote = compatDropPaths.some(p => OUTPUT_BUDGET_DROP_PATHS.includes(p)) + ? " (overrides the earlier 'output budget restored' log — the field is absent from the forwarded body)" + : ""; if (compatDropPaths.length > 0 && typeof wireBody === "string") { // #564/#2627: drops run on the SAME graph as every other transform, so // identity decides — not the drop path's shape. A drop that deletes or @@ -697,14 +704,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare const dropped = dropCompatFieldsJson(graph, compatDropPaths); if (dropped > 0) { commitGraph(); - log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")}) (#1757)`); + log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")})${budgetOverrideNote} (#1757)`); } } else { const applied = applyCompatDropFields(wireBody, compatDropPaths); if (applied.dropped > 0) { wireBody = applied.body; invalidateSeparatorOwnership(); - log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${applied.dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")}) (#1757)`); + log("info", `[${prepared?.session.id ?? "passthrough"}] [compat] dropped ${applied.dropped} field(s) per compat.dropFields (${compatDropPaths.join(", ")})${budgetOverrideNote} (#1757)`); } } } diff --git a/src/server/side-request.ts b/src/server/side-request.ts index b6c1cb04b0..0dfcf52252 100644 --- a/src/server/side-request.ts +++ b/src/server/side-request.ts @@ -6,6 +6,12 @@ import { type ResolvedImageBilling } from "../image-tokens.js"; import { ACP_TOOL_NAMES, ABSORB_TOOL_NAME, BLOCK_OP_TOOL_NAMES, RETRIEVE_TOOL_NAME, RULE_TOOL_NAME } from "acp-kernel"; import { READ_ORIGINAL_IMAGE_TOOL_NAME } from "../compress-tool.js"; import { dshCarrierRawMarkers } from "../durable-message-guards.js"; +import { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField } from "./budget.js"; + +// #6752: the output-budget field plumbing lives in budget.ts now (generic wire +// plumbing, not side-lane logic); re-exported here so existing importers +// (server.ts, tests) keep working unchanged. +export { outputBudgetField, readOutputBudget, writeOutputBudget, type OutputBudgetField }; // #388: side requests (title-gen etc.) share the main session key but must not // touch kernel state. Identified by a tiny output budget (same heuristic as @@ -607,49 +613,6 @@ export function stripLeakedBiliTools(parsed: unknown): boolean { return true; } -export type OutputBudgetField = "max_tokens" | "max_completion_tokens" | "max_output_tokens" | "generationConfig.maxOutputTokens"; - -/** The declared output budget, proto-agnostically. Gemini nests it under - * `generationConfig` (the dotted field name above), the OpenAI/Anthropic - * families keep it flat, so every reader goes through these two accessors. */ -export function outputBudgetField(parsed: unknown): OutputBudgetField | null { - if (!parsed || typeof parsed !== "object") return null; - const p = parsed as Record; - for (const field of ["max_tokens", "max_completion_tokens", "max_output_tokens"] as const) { - if (typeof p[field] === "number" && (p[field] as number) > 0) return field; - } - const gen = p.generationConfig; - if (gen && typeof gen === "object") { - const v = (gen as Record).maxOutputTokens; - if (typeof v === "number" && v > 0) return "generationConfig.maxOutputTokens"; - } - return null; -} - -export function readOutputBudget(parsed: Record, field: OutputBudgetField): number | undefined { - if (field !== "generationConfig.maxOutputTokens") { - const v = parsed[field]; - return typeof v === "number" ? v : undefined; - } - const gen = parsed.generationConfig; - if (!gen || typeof gen !== "object") return undefined; - const v = (gen as Record).maxOutputTokens; - return typeof v === "number" ? v : undefined; -} - -export function writeOutputBudget(parsed: Record, field: OutputBudgetField, value: number): void { - // #2011: output-budget fields are integer-typed on every wire protocol. Coerce at this - // sole writer so NO caller can emit a fractional token count — the clamp path already - // floors, but this also guards the restore path and any future caller at the type boundary. - const tokens = Number.isInteger(value) ? value : Math.floor(value); - if (field !== "generationConfig.maxOutputTokens") { - parsed[field] = tokens; - return; - } - const gen = parsed.generationConfig; - parsed.generationConfig = { ...(gen && typeof gen === "object" ? (gen as Record) : {}), maxOutputTokens: tokens }; -} - // #3071: absolute capability floor for the output budget when NO source knows a // ceiling for the model. The client's raw-history-derived budget decays unbounded // (…, 3071, 2061, 252, <=200); "last non-starved wins" then pins the session at a diff --git a/tests/issue6752-watermark-output-budget-restore.test.ts b/tests/issue6752-watermark-output-budget-restore.test.ts new file mode 100644 index 0000000000..0c5c7b0acd --- /dev/null +++ b/tests/issue6752-watermark-output-budget-restore.test.ts @@ -0,0 +1,333 @@ +// #6752 — output-budget restore lost on watermark hits. +// +// A dsh plugin-native turn with a client-shrunk output budget (max_tokens: 1) +// is rescued by restoreOutputBudget in handle.ts (log: "output budget +// restored 1 -> 32768"), but on a watermark-fastpath HIT prepareOpenai +// rebinds `parsed` from the RAW wire text (bodyText) — re-parsing every +// scalar field, including max_tokens — so the restored value never reaches +// the upstream while the log still reports success. +// +// A/B golden: the same two scripted turns run through two proxies — +// PHASE A: pluginFullResendWatermark OFF → legacy full-resend path +// PHASE B: rollout ON (env force) → fastpath (stamped hits) +// Turn 0 seeds the high-water (healthy 64000 budget) and the watermark; +// turn 1 carries the starved budget (max_tokens: 1) with tools present. +// Both phases must forward the RESTORED budget (32768, the declared output +// ceiling) on turn 1. Before the fix, phase B forwards max_tokens: 1. +// +// The residency check proves phase B actually HIT on turn 1 (a miss would +// take the legacy path and forward the restored value vacuously). + +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import { once } from "node:events"; +import os from "node:os"; +import path from "node:path"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { + SessionStore, + _setStoreForTest, + _resetStoreForTest, +} from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _resetSessionsForTest } from "../src/session.ts"; +import { closeServers } from "./server-close-guard.ts"; + +const ENV = "BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK"; + +interface Capture { + raw: string; +} +interface Harness2 { + root: string; + captured: Capture[]; + upstream: http.Server; + store: SessionStore; +} + +function makeUpstream(sink: Capture[]) { + return http.createServer((req, res) => { + let data = ""; + req.on("data", (c) => (data += c)); + req.on("end", () => { + sink.push({ raw: data }); + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + id: "chatcmpl-test", + object: "chat.completion", + created: 1, + model: "model-test", + choices: [ + { + index: 0, + message: { role: "assistant", content: "answer" }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 4, + total_tokens: 104, + }, + }), + ); + }); + }); +} + +function makeHarness(tag: string): Harness2 { + _resetStoreForTest(); + _resetPluginStateForTest(); + _resetSessionsForTest(); + const root = mkdtempSync(path.join(os.tmpdir(), `obw-${tag}-`)); + for (const k of [ + "HOME", + "APPHOME", + "HERMES_HOME", + "XDG_STATE_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + ]) + process.env[k] = root; + const store = new SessionStore({ + dir: path.join(root, "sessions"), + enabled: false, + }); + _setStoreForTest(store); + const captured: Capture[] = []; + return { root, store, captured, upstream: makeUpstream(captured) }; +} + +async function bringUp(h: Harness2) { + h.upstream.listen(0, "127.0.0.1"); + await once(h.upstream, "listening"); + const uaddr = h.upstream.address(); + assert(uaddr && typeof uaddr === "object"); + const upstreamUrl = `http://127.0.0.1:${uaddr.port}`; + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: upstreamUrl, + routes: { + [upstreamUrl]: { + models: { "model-test": { context: 64_000, output: 32_768 } }, + }, + }, + modelContextLimit: 200_000, + kernelConfig: defaultConfig(200_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: true, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const paddr = proxy.address(); + assert(paddr && typeof paddr === "object"); + return { proxy, origin: `http://127.0.0.1:${paddr.port}`, upstreamUrl }; +} + +function idOf(el: unknown): string { + const s = JSON.stringify(el); + let h = 0x811c9dc5; + for (let k = 0; k < s.length; k++) { + h ^= s.charCodeAt(k); + h = Math.imul(h, 0x01000193) >>> 0; + } + return `eid-${h.toString(16)}`; +} + +const TOOL = { + type: "function", + function: { + name: "test_tool", + description: "test tool", + parameters: { type: "object", properties: {} }, + }, +}; + +function scriptTurns(): Array<{ + model: string; + messages: unknown[]; + tools: unknown[]; + max_tokens: number; + stream: boolean; +}> { + // Turn 0: healthy budget + tools — main turn, seeds the high-water and + // the watermark. Turn 1: same history + one more exchange, starved + // budget (max_tokens: 1) + tools — restore must raise it to the declared + // output ceiling (32768) on BOTH paths. + const history: unknown[] = [ + { role: "system", content: "You are TestBuddy v2." }, + ]; + const lines = ["alpha", "beta", "gamma", "delta"]; + for (let i = 0; i < lines.length; i++) { + history.push({ role: "user", content: lines[i] }); + history.push({ role: "assistant", content: `ack ${i}` }); + } + const turn1 = [ + ...history, + { role: "user", content: "epsilon" }, + { role: "assistant", content: "ack 4" }, + { role: "user", content: "zeta" }, + ]; + return [ + { + model: "model-test", + messages: history, + tools: [TOOL], + max_tokens: 64_000, + stream: false, + }, + { + model: "model-test", + messages: turn1, + tools: [TOOL], + max_tokens: 1, + stream: false, + }, + ]; +} + +async function runClient( + origin: string, + upstreamUrl: string, + sid: string, + turns: ReturnType, +): Promise<(number | null)[]> { + const stamps: (number | null)[] = []; + for (let i = 0; i < turns.length; i++) { + const headers: Record = { + "content-type": "application/json", + "x-acp-session": sid, + "x-bili-plugin": "dsh", + }; + const prev = stamps[stamps.length - 1]; + if (prev !== null && prev !== undefined) + headers["x-bili-pf-count"] = String(prev); + const ids = (turns[i].messages as unknown[]).map(idOf); + if (prev !== null && prev !== undefined && ids.length >= prev) { + headers["x-bili-pf-tail-ids"] = ids + .slice(prev - 1) + .map(encodeURIComponent) + .join(","); + } + const res = await fetch( + `${origin}/bili/${upstreamUrl}/v1/chat/completions`, + { + method: "POST", + headers, + body: JSON.stringify(turns[i]), + }, + ); + assert.equal(res.status, 200, `turn ${i}: proxy answers 200`); + const s = res.headers.get("x-bili-pf-seq"); + stamps.push(s !== null && /^\d+$/.test(s) ? Number(s) : null); + await res.arrayBuffer(); + } + return stamps; +} + +test("#6752 watermark hit must not discard the restored output budget", async () => { + const savedEnv = process.env[ENV]; + const roots: string[] = []; + try { + // ---------- PHASE A: feature OFF (legacy full-resend) ---------- + process.env[ENV] = "0"; + const ha0 = makeHarness("phase-a"); + roots.push(ha0.root); + const ha = await bringUp(ha0); + const ra = await runClient( + ha.origin, + ha.upstreamUrl, + "obw-phase-a", + scriptTurns(), + ); + const memA = (await (await fetch(`${ha.origin}/__bili/memory`)).json()) as { + breakdown: { lastMessages: number }; + }; + ha0.store.cancelAll(); + await closeServers([ha.proxy, ha0.upstream], "obw phase-a"); + + // ---------- PHASE B: feature ON (fastpath stamped hits) ---------- + process.env[ENV] = "1"; + const hb0 = makeHarness("phase-b"); + roots.push(hb0.root); + const hb = await bringUp(hb0); + const rb = await runClient( + hb.origin, + hb.upstreamUrl, + "obw-phase-b", + scriptTurns(), + ); + const memB = (await (await fetch(`${hb.origin}/__bili/memory`)).json()) as { + breakdown: { lastMessages: number }; + }; + hb0.store.cancelAll(); + await closeServers([hb.proxy, hb0.upstream], "obw phase-b"); + + assert.equal(ha0.captured.length, 2, "legacy served both turns"); + assert.equal(hb0.captured.length, 2, "fastpath served both turns"); + // Turn 0 stamps (watermark seeded) in phase B; turn 1 must be a HIT, + // so the fastpath pins only the tail window (residency proof). + assert.equal( + rb[0], + scriptTurns()[0].messages.length, + "B stamp 0: measured wire total", + ); + assert.ok( + memA.breakdown.lastMessages > 0, + "legacy pinned resident history", + ); + assert.ok( + memB.breakdown.lastMessages > 0, + "fastpath pinned the tail window", + ); + assert.ok( + memB.breakdown.lastMessages < memA.breakdown.lastMessages, + `fastpath did not hit on turn 1: pinned ${memB.breakdown.lastMessages} vs legacy ${memA.breakdown.lastMessages}`, + ); + + const maxTokensOf = (raw: string): unknown => + (JSON.parse(raw) as { max_tokens?: unknown }).max_tokens; + // Turn 0 forwards the healthy budget CLAMPED to the declared output + // ceiling (32768) in both phases — clampOutgoingOutput caps at the + // ceiling a provider would accept; the restore is not involved here. + assert.equal( + maxTokensOf(ha0.captured[0].raw), + 32_768, + "A turn 0: healthy budget clamped to the ceiling", + ); + assert.equal( + maxTokensOf(hb0.captured[0].raw), + 32_768, + "B turn 0: healthy budget clamped to the ceiling", + ); + // Turn 1: the starved budget must be restored to the declared output + // ceiling (32768) on BOTH paths — the log already claims the restore. + assert.equal( + maxTokensOf(ha0.captured[1].raw), + 32_768, + "A turn 1: starved budget restored to the ceiling", + ); + assert.equal( + maxTokensOf(hb0.captured[1].raw), + 32_768, + "B turn 1: starved budget restored to the ceiling (watermark hit must not discard it)", + ); + void ra; + } finally { + if (savedEnv === undefined) delete process.env[ENV]; + else process.env[ENV] = savedEnv; + for (const root of roots) rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/side-lane-invariants.test.ts b/tests/side-lane-invariants.test.ts index 1d23a43f70..a4b48b5fe1 100644 --- a/tests/side-lane-invariants.test.ts +++ b/tests/side-lane-invariants.test.ts @@ -16,6 +16,8 @@ import { _resetPluginStateForTest, resolveConversation } from "../src/plugin.ts" import { _resetSessionsForTest, listSessions, SPLIT_CANARY_FRESH_MS, splitSessionWarnings, type Session } from "../src/session.ts"; import { _setForTest } from "../src/registry.ts"; import { resetToolRingForTest } from "../src/tool-ring.ts"; +import { outputBudgetField as budgetFieldFromBudget, readOutputBudget as readFromBudget, writeOutputBudget as writeFromBudget } from "../src/server/budget.ts"; +import { outputBudgetField as budgetFieldFromSide, readOutputBudget as readFromSide, writeOutputBudget as writeFromSide } from "../src/server/side-request.ts"; // #2170 measure 2: behavioral invariants of the #388 side lane, on the dsh // plugin wire (the host whose regression started this — #2156/#2157/#2164). @@ -329,6 +331,17 @@ test("(F) extracted protocol modules keep the side-lane boundary (#1440 P2 cut 2 } }); +// (#6752 / #1440 P2 cut 2): the output-budget field plumbing moved out of the +// side-lane engine into budget.ts so the prepare modules can use it without +// reaching into side-request.js. side-request.ts re-exports it for the existing +// importers — pin the identity so the re-export and the definition can never +// silently diverge (a broken re-export would break every importer in one step). +test("(F2) output-budget plumbing re-export identity (#6752 / #1440 P2 cut 2)", () => { + assert.equal(budgetFieldFromSide, budgetFieldFromBudget, "outputBudgetField re-export is identity"); + assert.equal(readFromSide, readFromBudget, "readOutputBudget re-export is identity"); + assert.equal(writeFromSide, writeFromBudget, "writeOutputBudget re-export is identity"); +}); + // (#1440 P2 cut 3): the extracted pipeline module OWNS the mid-pipeline lane // gates, so it legitimately imports side-request.js — unlike (F)'s modules. // What must not move or appear: the engine's internals (threshold constants,