Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 29 additions & 20 deletions src/loop/adapter-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,27 @@ function stripFinishReasonChunk(buf: Buffer): Buffer {
}
}

function patchUsageChunk(eventStr: string, parsed: Record<string, unknown>, u: Record<string, unknown>, hostCredit: number): Buffer {
const pu = typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined;
const tu = typeof u.total_tokens === "number" ? u.total_tokens : undefined;
if (hostCredit > 0 && (pu !== undefined || tu !== undefined)) {
const patched = {
...parsed,
usage: {
...u,
...(pu !== undefined ? { prompt_tokens: pu + hostCredit } : {}),
...(tu !== undefined ? { total_tokens: tu + hostCredit } : {}),
},
};
const out = eventStr
.split("\n")
.map((l) => (l.startsWith("data:") ? `data: ${JSON.stringify(patched)}` : l))
.join("\n");
return Buffer.from(out + "\n\n", "utf8");
}
return Buffer.from(eventStr + "\n\n", "utf8");
}

export function createOpenaiAdapter(requestBody: Record<string, unknown>, clientSystem?: string, hostCredit = 0): CompressLoopAdapter {
const model = (requestBody.model as string) ?? "unknown";
let responseId = `chatcmpl-proxy-${Date.now()}`;
Expand Down Expand Up @@ -311,6 +332,13 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
outputTokens: typeof u.completion_tokens === "number" ? u.completion_tokens : undefined,
cachedTokens: typeof pd?.cached_tokens === "number" ? pd.cached_tokens : undefined,
} as ParsedStreamEvent;
// #589: include_usage clients (dsh, OpenAI SDK) read usage
// from this trailing empty-choices frame; raw tool-call rounds
// must forward it (with the prepare-time credit), not swallow
// it into the internal ledger.
if (sawRealToolCall) {
yield { kind: "meta", chunk: patchUsageChunk(eventStr, parsed, u, hostCredit) } as ParsedStreamEvent;
}
}
continue;
}
Expand All @@ -334,26 +362,7 @@ export function createOpenaiAdapter(requestBody: Record<string, unknown>, client
// post-fold usage) reaches the host verbatim — add the
// prepare-time credit back so the host anchors on the
// uncompressed baseline.
let chunk = rawBuf;
if (hostCredit > 0 && u) {
const pu = typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined;
const tu = typeof u.total_tokens === "number" ? u.total_tokens : undefined;
if (pu !== undefined || tu !== undefined) {
const patched = {
...parsed,
usage: {
...u,
...(pu !== undefined ? { prompt_tokens: pu + hostCredit } : {}),
...(tu !== undefined ? { total_tokens: tu + hostCredit } : {}),
},
};
const out = eventStr
.split("\n")
.map((l) => (l.startsWith("data:") ? `data: ${JSON.stringify(patched)}` : l))
.join("\n");
chunk = Buffer.from(out + "\n\n", "utf8");
}
}
const chunk = patchUsageChunk(eventStr, parsed, u ?? {}, hostCredit);
// This verbatim chunk IS the round's authoritative completion
// (suppressCompletion); write it once and never fall through
// to the text/reasoning branches (which would re-emit the same
Expand Down
68 changes: 68 additions & 0 deletions tests/issue589-usage-frame.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { createOpenaiAdapter } from "../src/loop/index.ts";

function streamOf(events: string[]): ReadableStream<Uint8Array> {
let i = 0;
return new ReadableStream<Uint8Array>({
pull(controller) {
if (i < events.length) {
controller.enqueue(Buffer.from(events[i++], "utf8"));
} else {
controller.close();
}
},
});
}

const chunk = (o: unknown): string => `data: ${JSON.stringify(o)}\n\n`;

const toolDelta = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "get_weather", arguments: "" } }] } }] };
const finishChunk = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] };
const usageFrame = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [], usage: { prompt_tokens: 50000, completion_tokens: 10, total_tokens: 50010 } };

async function collect(adapter: ReturnType<typeof createOpenaiAdapter>, events: string[]): Promise<{ meta: string; kinds: string[] }> {
let meta = "";
const kinds: string[] = [];
for await (const ev of adapter.parseStream(streamOf(events), 1)) {
kinds.push(ev.kind);
if (ev.kind === "meta") meta += ev.chunk.toString("utf8");
}
return { meta, kinds };
}

test("#589: raw tool-call round forwards the trailing usage-only frame, patched with credit", async () => {
const adapter = createOpenaiAdapter({ model: "m" }, undefined, 40000);
const { meta } = await collect(adapter, [chunk(toolDelta), chunk(finishChunk), chunk(usageFrame), "data: [DONE]\n\n"]);
const usageAt = meta.indexOf('"prompt_tokens":90000');
const finishAt = meta.indexOf('"finish_reason":"tool_calls"');
const doneAt = meta.indexOf("[DONE]");
assert.ok(usageAt >= 0, `expected patched usage frame in client stream: ${meta}`);
assert.ok(meta.includes('"total_tokens":90010'), meta);
assert.ok(finishAt >= 0 && usageAt > finishAt, `usage frame must come after the finish chunk: ${meta}`);
assert.ok(doneAt >= 0 && usageAt < doneAt, `usage frame must come before [DONE]: ${meta}`);
});

test("#589: trailing usage-only frame forwarded verbatim when no credit", async () => {
const adapter = createOpenaiAdapter({ model: "m" }, undefined, 0);
const { meta } = await collect(adapter, [chunk(toolDelta), chunk(finishChunk), chunk(usageFrame), "data: [DONE]\n\n"]);
assert.ok(meta.includes('"prompt_tokens":50000'), meta);
assert.ok(!meta.includes('"prompt_tokens":90000'), meta);
});

test("#589: non-tool rounds keep swallowing the trailing frame (rebuild embeds usage)", async () => {
const adapter = createOpenaiAdapter({ model: "m" }, undefined, 40000);
const textDelta = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: { content: "hi there" } }] };
const finishText = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] };
const { meta } = await collect(adapter, [chunk(textDelta), chunk(finishText), chunk(usageFrame), "data: [DONE]\n\n"]);
assert.ok(!meta.includes("prompt_tokens"), `rebuild round must not forward the raw usage frame: ${meta}`);
});

test("#589: usage-only frame without usage field is not forwarded", async () => {
const adapter = createOpenaiAdapter({ model: "m" }, undefined, 40000);
const emptyChoices = { id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [] };
const { meta } = await collect(adapter, [chunk(toolDelta), chunk(finishChunk), chunk(emptyChoices), "data: [DONE]\n\n"]);
assert.ok(!meta.includes("chat.completion.chunk\" }"), meta);
const frames = meta.split("data: ").length - 1;
assert.equal(frames, 3, `expected only tool-delta + finish + [DONE] frames: ${meta}`);
});
Loading