Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions scripts/bench-stream-frame-scan.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { performance } from "node:perf_hooks";
import { createGoogleAdapter } from "../src/loop/adapter-google.ts";

const chunkSize = 16 * 1024;
const sizes = process.argv.slice(2).map(Number);
for (const size of sizes.length ? sizes : [1, 4, 8]) {
const payload = "A".repeat(size * 1024 * 1024);
const wire = Buffer.from(`data: ${JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "Generated image." }, { inlineData: { mimeType: "image/png", data: payload } }] } }] })}\n\ndata: ${JSON.stringify({ candidates: [{ finishReason: "STOP" }] })}\n\n`);
const times = [];
for (let pass = 0; pass < 4; pass++) {
let at = 0;
const stream = new ReadableStream({ pull(controller) {
if (at >= wire.length) return controller.close();
controller.enqueue(wire.subarray(at, at + chunkSize));
at += chunkSize;
} });
const adapter = createGoogleAdapter({}, undefined, undefined, "gemini-test");
let outputBytes = 0;
const start = performance.now();
for await (const event of adapter.parseStream(stream, 1)) {
if (event.kind === "meta") outputBytes += event.chunk.length;
else if ("raw" in event && event.raw) outputBytes += event.raw.length;
}
const elapsed = performance.now() - start;
if (outputBytes < payload.length) throw new Error("Image frame was not forwarded");
if (pass > 0) times.push(elapsed);
}
times.sort((a, b) => a - b);
console.log(JSON.stringify({ payloadMiB: size, wireBytes: wire.length, networkChunkBytes: chunkSize, medianMs: Number(times[1].toFixed(2)) }));
}
18 changes: 4 additions & 14 deletions src/loop/adapter-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { buildVisibilityMarker } from "./core.js";
import { composeStreamFilters, createBiliArtifactFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js";
import { degenerateTurnWarning } from "../degenerate-turn.js";
import { log as loggerLog } from "../logger.js";
import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData, replaceSseEventData } from "../sse-util.js";
import { createSseFramer, sseEventData, replaceSseEventData } from "../sse-util.js";
import type {
CompressLoopAdapter,
EmitCompletionOpts,
Expand All @@ -24,7 +24,7 @@ interface ToolUseBuffer {
async function* iterSseEvents(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buf = "";
const framer = createSseFramer();
try {
while (true) {
let done: boolean;
Expand All @@ -35,23 +35,13 @@ async function* iterSseEvents(stream: ReadableStream<Uint8Array>): AsyncGenerato
break;
}
if (done) break;
buf += decoder.decode(value, { stream: true });
buf = normalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.push(decoder.decode(value, { stream: true }))) {
if (raw.trim().length > 0) yield raw;
}
}
buf = finalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.finish()) {
if (raw.trim().length > 0) yield raw;
}
if (buf.trim().length > 0) yield buf;
} finally {
reader.releaseLock();
}
Expand Down
20 changes: 5 additions & 15 deletions src/loop/adapter-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { buildVisibilityMarker } from "./core.js";
import { composeStreamFilters, createBiliArtifactFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js";
import { degenerateTurnWarning } from "../degenerate-turn.js";
import { log as loggerLog } from "../logger.js";
import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData } from "../sse-util.js";
import { createSseFramer, sseEventData } from "../sse-util.js";

import type {
CompressLoopAdapter,
Expand Down Expand Up @@ -169,6 +169,7 @@ interface GoogleChunk {
async function* iterGoogleChunks(stream: ReadableStream<Uint8Array>): AsyncGenerator<GoogleChunk> {
const reader = stream.getReader();
const decoder = new TextDecoder();
const framer = createSseFramer();
let buf = "";
let mode: "unknown" | "sse" | "array" = "unknown";
let cursor = 0;
Expand Down Expand Up @@ -200,14 +201,11 @@ async function* iterGoogleChunks(stream: ReadableStream<Uint8Array>): AsyncGener
}
}
if (mode === "sse") {
buf = normalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.push(buf)) {
const json = dataLine(raw);
if (json !== null) yield { frame: raw, json };
}
buf = "";
continue;
}
for (; cursor < buf.length && !arrayDone; cursor++) {
Expand Down Expand Up @@ -241,18 +239,10 @@ async function* iterGoogleChunks(stream: ReadableStream<Uint8Array>): AsyncGener
}
}
if (mode === "sse") {
buf = finalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.finish()) {
const json = dataLine(raw);
if (json !== null) yield { frame: raw, json };
}
if (buf.trim().length > 0) {
const json = dataLine(buf);
if (json !== null) yield { frame: buf, json };
}
}
} finally {
reader.releaseLock();
Expand Down
18 changes: 4 additions & 14 deletions src/loop/adapter-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { buildVisibilityMarker } from "./core.js";
import { composeStreamFilters, createBiliArtifactFilter, createIdentityStreamFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js";
import { degenerateTurnWarning } from "../degenerate-turn.js";
import { log as loggerLog } from "../logger.js";
import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData, replaceSseEventData } from "../sse-util.js";
import { createSseFramer, sseEventData, replaceSseEventData } from "../sse-util.js";
import { hardenOpenaiAssistantContent, systemToUser } from "../util.js";

import type {
Expand Down Expand Up @@ -87,7 +87,7 @@ interface ToolCallBuffer {
async function* iterSseChunks(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buf = "";
const framer = createSseFramer();
try {
while (true) {
let done: boolean;
Expand All @@ -98,23 +98,13 @@ async function* iterSseChunks(stream: ReadableStream<Uint8Array>): AsyncGenerato
break;
}
if (done) break;
buf += decoder.decode(value, { stream: true });
buf = normalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.push(decoder.decode(value, { stream: true }))) {
if (raw.trim().length > 0) yield raw;
}
}
buf = finalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.finish()) {
if (raw.trim().length > 0) yield raw;
}
if (buf.trim().length > 0) yield buf;
} finally {
reader.releaseLock();
}
Expand Down
18 changes: 4 additions & 14 deletions src/loop/adapter-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { hashId, strippedResponseIdWarning } from "../util.js";
import { composeStreamFilters, createBiliArtifactFilter, createIdentityStreamFilter, createMarkerLineFilter, createTagEchoFilter, stripResponsesText, containsBiliInternalText, containsMarkerLineText, containsRenderTagText, containsToolCallEmissionText, ACP_NAME_ALT, type TagEchoFilter } from "./tag-echo-filter.js";
import { degenerateTurnWarning } from "../degenerate-turn.js";
import { log as loggerLog } from "../logger.js";
import { normalizeSseLineEndings, finalizeSseLineEndings } from "../sse-util.js";
import { createSseFramer } from "../sse-util.js";
import { extractResponsesTextTriggers, PROXY_TOOL_NAMES } from "../compress-tool.js";
import type { BiliMessage } from "acp-kernel/wire";
import type {
Expand Down Expand Up @@ -182,7 +182,7 @@ function rebuildResponsesEvent(type: string, obj: Record<string, unknown>): Buff
async function* iterSseEvents(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buf = "";
const framer = createSseFramer();
try {
while (true) {
let done: boolean;
Expand All @@ -193,23 +193,13 @@ async function* iterSseEvents(stream: ReadableStream<Uint8Array>): AsyncGenerato
break;
}
if (done) break;
buf += decoder.decode(value, { stream: true });
buf = normalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.push(decoder.decode(value, { stream: true }))) {
if (raw.trim().length > 0) yield raw;
}
}
buf = finalizeSseLineEndings(buf);
let idx: number;
while ((idx = buf.indexOf("\n\n")) >= 0) {
const raw = buf.slice(0, idx);
buf = buf.slice(idx + 2);
for (const raw of framer.finish()) {
if (raw.trim().length > 0) yield raw;
}
if (buf.trim().length > 0) yield buf;
} finally {
reader.releaseLock();
}
Expand Down
50 changes: 50 additions & 0 deletions src/sse-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,56 @@ export function finalizeSseLineEndings(buf: string): string {
return buf.replace(/\r\n|\r/g, "\n");
}

/** Frame newly decoded chunks without repeatedly scanning an unfinished
* event. Normalize only the new chunk plus a possible split CR, and join
* incomplete line fragments once when that line ends. Frames preserve the
* existing LF-normalized `split("\n\n")` semantics, including EOF residuals. */
export function createSseFramer(): {
push(chunk: string): Generator<string>;
finish(): Generator<string>;
} {
let pendingCr = "";
let lineParts: string[] = [];
let frameParts: string[] = [];

function* push(chunk: string): Generator<string> {
let text = normalizeSseLineEndings(pendingCr + chunk);
pendingCr = text.endsWith("\r") ? "\r" : "";
if (pendingCr) text = text.slice(0, -1);
let start = 0;
let end: number;
while ((end = text.indexOf("\n", start)) !== -1) {
if (end > start) lineParts.push(text.slice(start, end));
if (lineParts.length === 0 && frameParts.length > 0) {
// The preceding line's LF and this blank line's LF are the
// separator; neither belongs to the returned raw frame.
frameParts.pop();
const frame = frameParts.join("");
frameParts = [];
yield frame;
} else {
if (lineParts.length > 0) frameParts.push(lineParts.join(""));
frameParts.push("\n");
}
lineParts = [];
start = end + 1;
}
if (start < text.length) lineParts.push(text.slice(start));
}

function* finish(): Generator<string> {
// A final lone CR is one terminator. Resolving it with an LF uses
// the same split-CRLF path and contributes exactly one normalized LF.
if (pendingCr) yield* push("\n");
const tail = [...frameParts, ...lineParts].join("");
frameParts = [];
lineParts = [];
if (tail.length > 0) yield tail;
}

return { push, finish };
}

/** Data fields of one already-framed, LF-normalized SSE event form ONE
* payload, joined by LF. Remove only the optional first space after `:`;
* indentation and empty fields are part of the payload. */
Expand Down
95 changes: 95 additions & 0 deletions tests/loop-sse-framing-scan.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import assert from "node:assert/strict";
import test from "node:test";
import { createGoogleAdapter } from "../src/loop/adapter-google.ts";
import { createSseFramer, finalizeSseLineEndings } from "../src/sse-util.ts";

function chunks(wire: Buffer, chunkSize: number): ReadableStream<Uint8Array> {
let offset = 0;
return new ReadableStream({ pull(controller) {
if (offset >= wire.length) { controller.close(); return; }
controller.enqueue(wire.subarray(offset, offset + chunkSize));
offset += chunkSize;
} });
}

function googleFrame(payload: string): string {
return `data: ${JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "Generated image." }, { inlineData: { mimeType: "image/png", data: payload } }] } }] })}\n\n`;
}

async function googleOutput(wire: Buffer, chunkSize: number): Promise<string> {
const adapter = createGoogleAdapter({}, undefined, undefined, "gemini-test");
const parts: Buffer[] = [];
for await (const event of adapter.parseStream(chunks(wire, chunkSize), 1)) {
if (event.kind === "meta") parts.push(event.chunk);
else if ("raw" in event && event.raw) parts.push(event.raw);
}
return Buffer.concat(parts).toString("utf8");
}

test("Google loop framing scans newly received bytes, not the accumulated image prefix", async (t) => {
const frame = googleFrame("A".repeat(1024 * 1024));
const wire = Buffer.from(frame);
const indexOf = String.prototype.indexOf;
let searchedChars = 0;
t.mock.method(String.prototype, "indexOf", function (this: string, needle: string, position?: number): number {
const value = String(this);
const result = indexOf.call(value, needle, position);
if (needle === "\r" || needle === "\n\n" || needle === "\n") {
const start = Math.max(0, position ?? 0);
searchedChars += Math.max(0, (result < 0 ? value.length : result + needle.length) - start);
}
return result;
});
assert.equal(await googleOutput(wire, 4 * 1024), frame, "the complete image/text frame stays byte-exact");
t.diagnostic(`SSE searches examined ${searchedChars} characters for ${wire.length} received bytes`);
assert.ok(searchedChars <= wire.length * 8, `SSE searches examined ${searchedChars} characters for ${wire.length} received bytes`);
});

for (const ending of ["\n", "\r\n", "\r"]) {
test(`Google loop framing retains Unicode across one-byte ${JSON.stringify(ending)} splits`, async () => {
const normalized = googleFrame("你好🔥");
const wire = Buffer.from(normalized.replaceAll("\n", ending));
assert.equal(await googleOutput(wire, 1), normalized);
});
}

function frameChunks(parts: string[]): string[] {
const framer = createSseFramer();
return [...parts.flatMap(part => [...framer.push(part)]), ...framer.finish()];
}

function wholeFrames(text: string): string[] {
const frames = finalizeSseLineEndings(text).split("\n\n");
if (frames.at(-1) === "") frames.pop();
return frames;
}

test("incremental framing preserves complete-field whitespace and EOF residuals at every split", () => {
const inputs = [
"", "\n", "\r", "\r\n", "\n\n\n", "\n\ndata: tail", "\ndata: tail\n",
":comment\r\nid: 1\r\nevent: update\r\ndata: {\r\ndata: \"text\": \"你好🔥\"\r\ndata: }\r\n\r\n",
"data\ndata: indented\ndata:\n\ndata: residual\n",
"event: partial\rdata: {\"text\":\"tail\"}\r",
];
for (const input of inputs) {
const expected = wholeFrames(input);
for (let cut = 0; cut <= input.length; cut++) {
assert.deepEqual(frameChunks([input.slice(0, cut), input.slice(cut)]), expected, `split ${cut}: ${JSON.stringify(input)}`);
}
assert.deepEqual(frameChunks(input.split("")), expected, `one-character chunks: ${JSON.stringify(input)}`);
}
});

test("all short CR/LF combinations match the previous whole-buffer framing", () => {
const alphabet = ["a", "\r", "\n"];
let values = [""];
for (let length = 0; length <= 6; length++) {
for (const value of values) {
const expected = wholeFrames(value);
for (let cut = 0; cut <= value.length; cut++) {
assert.deepEqual(frameChunks([value.slice(0, cut), value.slice(cut)]), expected);
}
}
values = values.flatMap(value => alphabet.map(char => value + char));
}
});
Loading