diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index a195352343..4a14d2173a 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -166,8 +166,24 @@ function openaiCachedTokens(u: Record): number | null { export function createOpenaiAdapter(requestBody: Record, clientSystem?: string, absorbName?: string, notes?: string[], errorShape: "protocol" | "completion" = "protocol"): CompressLoopAdapter { const model = (requestBody.model as string) ?? "unknown"; + // OpenAI clients opt into the trailing usage frame with + // stream_options.include_usage. bili forces the upstream option on for its + // own ledger, but the frame must also reach the CLIENT when it asked for + // it: cost/context-aware clients read provider fields from that frame, and + // swallowing it on plain-text rounds silently loses them. Tool-call rounds + // keep the existing verbatim forward. + const clientWantsUsage = (() => { + const so = (requestBody as { stream_options?: unknown }).stream_options; + return typeof so === "object" && so !== null + && (so as Record).include_usage === true; + })(); let responseId = `chatcmpl-proxy-${Date.now()}`; let toolIndex = 0; + // Last upstream usage object seen on the wire. The synthesized completion + // frame re-states tokens; provider-specific fields (cost/credits, cache + // details, validity flags) must survive that re-statement — dropping them + // breaks any client that prices or accounts the turn from them. + let lastUsageRaw: Record | null = null; const makeBase = () => ({ id: responseId, @@ -251,6 +267,9 @@ export function createOpenaiAdapter(requestBody: Record, client }, async *parseStream(upstream, round) { + // Per-round scope: a later round that emits no usage frame must not + // inherit the previous round's provider fields (e.g. `credits`). + lastUsageRaw = null; const pending = new Map(); // #206: strip model-imitated render tags from PROSE deltas; the // filter may hold back a short tail, flushed at finish/[DONE]. @@ -506,6 +525,7 @@ export function createOpenaiAdapter(requestBody: Record, client if (!choice) { if (parsed.usage) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -516,7 +536,7 @@ export function createOpenaiAdapter(requestBody: Record, client // from this trailing empty-choices frame; raw tool-call rounds // must forward it verbatim, not swallow it into the internal // ledger. - if (sawRealToolCall) { + if (sawRealToolCall || clientWantsUsage) { yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent; } } @@ -549,6 +569,7 @@ export function createOpenaiAdapter(requestBody: Record, client // only the ledger gains the sample. if (parsed.usage && !finishReason && delta && Object.keys(delta).length === 0) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -562,6 +583,7 @@ export function createOpenaiAdapter(requestBody: Record, client const hadToolCalls = [...pending.values()].some((tc) => tc.name.length > 0 || tc.id.length > 0); yield* settleToolCalls(); const u = parsed.usage as Record | undefined; + if (u) lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u?.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -692,13 +714,18 @@ export function createOpenaiAdapter(requestBody: Record, client // Numeric fields must never serialize to absent/undefined: strict // clients (dsh's mapUsage) compute over prompt_tokens/completion_tokens // and a usage object missing them yields NaN (non-JSON-serializable). + const raw = lastUsageRaw ?? {}; + const rawDetails = raw.prompt_tokens_details && typeof raw.prompt_tokens_details === "object" + ? raw.prompt_tokens_details as Record + : undefined; const usage = opts?.usage ? { + ...raw, prompt_tokens: opts.usage.inputTokens ?? 0, completion_tokens: opts.usage.outputTokens ?? 0, total_tokens: (opts.usage.inputTokens ?? 0) + (opts.usage.outputTokens ?? 0), ...(typeof opts.usage.cachedTokens === "number" - ? { prompt_tokens_details: { cached_tokens: opts.usage.cachedTokens } } + ? { prompt_tokens_details: { ...rawDetails, cached_tokens: opts.usage.cachedTokens } } : {}), } : null; diff --git a/tests/openai-usage-frame-forward.test.ts b/tests/openai-usage-frame-forward.test.ts new file mode 100644 index 0000000000..6052dd73d4 --- /dev/null +++ b/tests/openai-usage-frame-forward.test.ts @@ -0,0 +1,104 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createOpenaiAdapter } from "../src/loop/index.ts"; +import type { ParsedStreamEvent } from "../src/loop/core.ts"; + +// Regression: a plain-text OpenAI turn with stream_options.include_usage must +// forward the trailing usage-only frame to the client. bili used to swallow it +// unless a tool call was seen, so clients that price the turn from that frame +// (IBM Bob) reported cost 0 while the account was still charged. + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +const frame = (obj: Record) => enc.encode(`data: ${JSON.stringify(obj)}\n\n`); + +const textTurn = (...extra: Uint8Array[]): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + for (const e of extra) controller.enqueue(e); + controller.enqueue(enc.encode("data: [DONE]\n\n")); + controller.close(); + }, + }); + +const usageFrame = (prompt: number, completion: number) => + frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [], + usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion }, + }); + +const collect = async (body: Record, stream: ReadableStream) => { + const adapter = createOpenaiAdapter(body); + const events: ParsedStreamEvent[] = []; + for await (const ev of adapter.parseStream(stream, 1)) events.push(ev); + return events; +}; + +const wire = (events: ParsedStreamEvent[]): string => + events + .filter((ev) => ev.kind === "meta") + .map((ev) => dec.decode((ev as { chunk: Uint8Array }).chunk)) + .join(""); + +test("include_usage client receives the trailing usage-only frame on a text turn", async () => { + const events = await collect( + { model: "gpt", stream: true, stream_options: { include_usage: true } }, + textTurn(usageFrame(123, 4)), + ); + const out = wire(events); + assert.match(out, /"usage":\{"prompt_tokens":123/); + assert.match(out, /"completion_tokens":4/); +}); + +test("client that did not opt in does not receive the usage-only frame", async () => { + const events = await collect({ model: "gpt", stream: true }, textTurn(usageFrame(123, 4))); + assert.doesNotMatch(wire(events), /"prompt_tokens"/); +}); + +test("tool-call round still forwards the usage frame verbatim", async () => { + const toolCall = frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }] }, finish_reason: "tool_calls" }], + }); + const events = await collect({ model: "gpt", stream: true }, textTurn(toolCall, usageFrame(50, 2))); + assert.match(wire(events), /"prompt_tokens":50/); +}); + +test("synthesized completion preserves provider-specific usage fields (credits)", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const usageWithCredits = frame({ + choices: [{ delta: {}, index: 0 }], + created: 1, + id: "c1", + model: "gpt", + object: "chat.completion.chunk", + usage: { prompt_tokens: 123, completion_tokens: 4, total_tokens: 127, credits: 0.42, cache_validity: false }, + }); + for await (const _ of adapter.parseStream(textTurn(usageWithCredits), 1)) { /* drain */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 123, outputTokens: 4, cachedTokens: 0 } }).toString("utf8"); + assert.match(completion, /"credits":0.42/); + assert.match(completion, /"prompt_tokens":123/); +}); + +test("a round without usage does not inherit the previous round's credits", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const withCredits = frame({ + choices: [{ delta: {}, index: 0 }], created: 1, id: "c1", model: "gpt", object: "chat.completion.chunk", + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12, credits: 0.42 }, + }); + for await (const _ of adapter.parseStream(textTurn(withCredits), 1)) { /* drain */ } + for await (const _ of adapter.parseStream(textTurn(), 2)) { /* drain, no usage frame */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 10, outputTokens: 2, cachedTokens: 0 } }).toString("utf8"); + assert.doesNotMatch(completion, /"credits"/); +});