Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions src/loop/adapter-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,24 @@ function openaiCachedTokens(u: Record<string, unknown>): number | null {

export function createOpenaiAdapter(requestBody: Record<string, unknown>, clientSystem?: string, absorbName?: string, notes?: string[], errorShape: "protocol" | "completion" = "protocol"): CompressLoopAdapter {
const model = (requestBody.model as string) ?? "unknown";
// OpenAI clients opt into the trailing usage frame with
// stream_options.include_usage. bili forces the upstream option on for its
// own ledger, but the frame must also reach the CLIENT when it asked for
// it: cost/context-aware clients read provider fields from that frame, and
// swallowing it on plain-text rounds silently loses them. Tool-call rounds
// keep the existing verbatim forward.
const clientWantsUsage = (() => {
const so = (requestBody as { stream_options?: unknown }).stream_options;
return typeof so === "object" && so !== null
&& (so as Record<string, unknown>).include_usage === true;
})();
let responseId = `chatcmpl-proxy-${Date.now()}`;
let toolIndex = 0;
// Last upstream usage object seen on the wire. The synthesized completion
// frame re-states tokens; provider-specific fields (cost/credits, cache
// details, validity flags) must survive that re-statement — dropping them
// breaks any client that prices or accounts the turn from them.
let lastUsageRaw: Record<string, unknown> | null = null;

const makeBase = () => ({
id: responseId,
Expand Down Expand Up @@ -251,6 +267,9 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
},

async *parseStream(upstream, round) {
// Per-round scope: a later round that emits no usage frame must not
// inherit the previous round's provider fields (e.g. `credits`).
lastUsageRaw = null;
const pending = new Map<number, ToolCallBuffer>();
// #206: strip model-imitated render tags from PROSE deltas; the
// filter may hold back a short tail, flushed at finish/[DONE].
Expand Down Expand Up @@ -506,6 +525,7 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
if (!choice) {
if (parsed.usage) {
const u = parsed.usage as Record<string, unknown>;
lastUsageRaw = u;
yield {
kind: "usage",
inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined,
Expand All @@ -516,7 +536,7 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
// from this trailing empty-choices frame; raw tool-call rounds
// must forward it verbatim, not swallow it into the internal
// ledger.
if (sawRealToolCall) {
if (sawRealToolCall || clientWantsUsage) {
yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent;
}
}
Expand Down Expand Up @@ -549,6 +569,7 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
// only the ledger gains the sample.
if (parsed.usage && !finishReason && delta && Object.keys(delta).length === 0) {
const u = parsed.usage as Record<string, unknown>;
lastUsageRaw = u;
yield {
kind: "usage",
inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined,
Expand All @@ -562,6 +583,7 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
const hadToolCalls = [...pending.values()].some((tc) => tc.name.length > 0 || tc.id.length > 0);
yield* settleToolCalls();
const u = parsed.usage as Record<string, unknown> | undefined;
if (u) lastUsageRaw = u;
yield {
kind: "usage",
inputTokens: typeof u?.prompt_tokens === "number" ? u.prompt_tokens : undefined,
Expand Down Expand Up @@ -692,13 +714,18 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
// Numeric fields must never serialize to absent/undefined: strict
// clients (dsh's mapUsage) compute over prompt_tokens/completion_tokens
// and a usage object missing them yields NaN (non-JSON-serializable).
const raw = lastUsageRaw ?? {};
const rawDetails = raw.prompt_tokens_details && typeof raw.prompt_tokens_details === "object"
? raw.prompt_tokens_details as Record<string, unknown>
: undefined;
const usage = opts?.usage
? {
...raw,
prompt_tokens: opts.usage.inputTokens ?? 0,
completion_tokens: opts.usage.outputTokens ?? 0,
total_tokens: (opts.usage.inputTokens ?? 0) + (opts.usage.outputTokens ?? 0),
...(typeof opts.usage.cachedTokens === "number"
? { prompt_tokens_details: { cached_tokens: opts.usage.cachedTokens } }
? { prompt_tokens_details: { ...rawDetails, cached_tokens: opts.usage.cachedTokens } }
: {}),
}
: null;
Expand Down
104 changes: 104 additions & 0 deletions tests/openai-usage-frame-forward.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { createOpenaiAdapter } from "../src/loop/index.ts";
import type { ParsedStreamEvent } from "../src/loop/core.ts";

// Regression: a plain-text OpenAI turn with stream_options.include_usage must
// forward the trailing usage-only frame to the client. bili used to swallow it
// unless a tool call was seen, so clients that price the turn from that frame
// (IBM Bob) reported cost 0 while the account was still charged.

const enc = new TextEncoder();
const dec = new TextDecoder();

const frame = (obj: Record<string, unknown>) => enc.encode(`data: ${JSON.stringify(obj)}\n\n`);

const textTurn = (...extra: Uint8Array[]): ReadableStream<Uint8Array> =>
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] }));
controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] }));
controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }));
for (const e of extra) controller.enqueue(e);
controller.enqueue(enc.encode("data: [DONE]\n\n"));
controller.close();
},
});

const usageFrame = (prompt: number, completion: number) =>
frame({
id: "c1",
object: "chat.completion.chunk",
created: 1,
model: "gpt",
choices: [],
usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion },
});

const collect = async (body: Record<string, unknown>, stream: ReadableStream<Uint8Array>) => {
const adapter = createOpenaiAdapter(body);
const events: ParsedStreamEvent[] = [];
for await (const ev of adapter.parseStream(stream, 1)) events.push(ev);
return events;
};

const wire = (events: ParsedStreamEvent[]): string =>
events
.filter((ev) => ev.kind === "meta")
.map((ev) => dec.decode((ev as { chunk: Uint8Array }).chunk))
.join("");

test("include_usage client receives the trailing usage-only frame on a text turn", async () => {
const events = await collect(
{ model: "gpt", stream: true, stream_options: { include_usage: true } },
textTurn(usageFrame(123, 4)),
);
const out = wire(events);
assert.match(out, /"usage":\{"prompt_tokens":123/);
assert.match(out, /"completion_tokens":4/);
});

test("client that did not opt in does not receive the usage-only frame", async () => {
const events = await collect({ model: "gpt", stream: true }, textTurn(usageFrame(123, 4)));
assert.doesNotMatch(wire(events), /"prompt_tokens"/);
});

test("tool-call round still forwards the usage frame verbatim", async () => {
const toolCall = frame({
id: "c1",
object: "chat.completion.chunk",
created: 1,
model: "gpt",
choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }] }, finish_reason: "tool_calls" }],
});
const events = await collect({ model: "gpt", stream: true }, textTurn(toolCall, usageFrame(50, 2)));
assert.match(wire(events), /"prompt_tokens":50/);
});

test("synthesized completion preserves provider-specific usage fields (credits)", async () => {
const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } });
const usageWithCredits = frame({
choices: [{ delta: {}, index: 0 }],
created: 1,
id: "c1",
model: "gpt",
object: "chat.completion.chunk",
usage: { prompt_tokens: 123, completion_tokens: 4, total_tokens: 127, credits: 0.42, cache_validity: false },
});
for await (const _ of adapter.parseStream(textTurn(usageWithCredits), 1)) { /* drain */ }
const completion = adapter.emitCompletion({ usage: { inputTokens: 123, outputTokens: 4, cachedTokens: 0 } }).toString("utf8");
assert.match(completion, /"credits":0.42/);
assert.match(completion, /"prompt_tokens":123/);
});

test("a round without usage does not inherit the previous round's credits", async () => {
const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } });
const withCredits = frame({
choices: [{ delta: {}, index: 0 }], created: 1, id: "c1", model: "gpt", object: "chat.completion.chunk",
usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12, credits: 0.42 },
});
for await (const _ of adapter.parseStream(textTurn(withCredits), 1)) { /* drain */ }
for await (const _ of adapter.parseStream(textTurn(), 2)) { /* drain, no usage frame */ }
const completion = adapter.emitCompletion({ usage: { inputTokens: 10, outputTokens: 2, cachedTokens: 0 } }).toString("utf8");
assert.doesNotMatch(completion, /"credits"/);
});
Loading