From 7a1f401b6bb073193e11f8a8cfe6d34651094026 Mon Sep 17 00:00:00 2001 From: Afloat Date: Sun, 15 Nov 2026 23:24:53 +0800 Subject: [PATCH] perf: frame loop SSE chunks in linear time Preserve existing newline and EOF semantics while scanning only newly received text in the four SSE loop adapters. Fixes #6849 --- scripts/bench-stream-frame-scan.mjs | 30 +++++++++ src/loop/adapter-anthropic.ts | 18 ++---- src/loop/adapter-google.ts | 20 ++---- src/loop/adapter-openai.ts | 18 ++---- src/loop/adapter-responses.ts | 18 ++---- src/sse-util.ts | 50 +++++++++++++++ tests/loop-sse-framing-scan.test.ts | 95 +++++++++++++++++++++++++++++ 7 files changed, 192 insertions(+), 57 deletions(-) create mode 100644 scripts/bench-stream-frame-scan.mjs create mode 100644 tests/loop-sse-framing-scan.test.ts diff --git a/scripts/bench-stream-frame-scan.mjs b/scripts/bench-stream-frame-scan.mjs new file mode 100644 index 0000000000..eeb8d51648 --- /dev/null +++ b/scripts/bench-stream-frame-scan.mjs @@ -0,0 +1,30 @@ +import { performance } from "node:perf_hooks"; +import { createGoogleAdapter } from "../src/loop/adapter-google.ts"; + +const chunkSize = 16 * 1024; +const sizes = process.argv.slice(2).map(Number); +for (const size of sizes.length ? sizes : [1, 4, 8]) { + const payload = "A".repeat(size * 1024 * 1024); + const wire = Buffer.from(`data: ${JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "Generated image." }, { inlineData: { mimeType: "image/png", data: payload } }] } }] })}\n\ndata: ${JSON.stringify({ candidates: [{ finishReason: "STOP" }] })}\n\n`); + const times = []; + for (let pass = 0; pass < 4; pass++) { + let at = 0; + const stream = new ReadableStream({ pull(controller) { + if (at >= wire.length) return controller.close(); + controller.enqueue(wire.subarray(at, at + chunkSize)); + at += chunkSize; + } }); + const adapter = createGoogleAdapter({}, undefined, undefined, "gemini-test"); + let outputBytes = 0; + const start = performance.now(); + for await (const event of adapter.parseStream(stream, 1)) { + if (event.kind === "meta") outputBytes += event.chunk.length; + else if ("raw" in event && event.raw) outputBytes += event.raw.length; + } + const elapsed = performance.now() - start; + if (outputBytes < payload.length) throw new Error("Image frame was not forwarded"); + if (pass > 0) times.push(elapsed); + } + times.sort((a, b) => a - b); + console.log(JSON.stringify({ payloadMiB: size, wireBytes: wire.length, networkChunkBytes: chunkSize, medianMs: Number(times[1].toFixed(2)) })); +} diff --git a/src/loop/adapter-anthropic.ts b/src/loop/adapter-anthropic.ts index 0ed396aa03..468f08e675 100644 --- a/src/loop/adapter-anthropic.ts +++ b/src/loop/adapter-anthropic.ts @@ -7,7 +7,7 @@ import { buildVisibilityMarker } from "./core.js"; import { composeStreamFilters, createBiliArtifactFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js"; import { degenerateTurnWarning } from "../degenerate-turn.js"; import { log as loggerLog } from "../logger.js"; -import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData, replaceSseEventData } from "../sse-util.js"; +import { createSseFramer, sseEventData, replaceSseEventData } from "../sse-util.js"; import type { CompressLoopAdapter, EmitCompletionOpts, @@ -24,7 +24,7 @@ interface ToolUseBuffer { async function* iterSseEvents(stream: ReadableStream): AsyncGenerator { const reader = stream.getReader(); const decoder = new TextDecoder(); - let buf = ""; + const framer = createSseFramer(); try { while (true) { let done: boolean; @@ -35,23 +35,13 @@ async function* iterSseEvents(stream: ReadableStream): AsyncGenerato break; } if (done) break; - buf += decoder.decode(value, { stream: true }); - buf = normalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.push(decoder.decode(value, { stream: true }))) { if (raw.trim().length > 0) yield raw; } } - buf = finalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.finish()) { if (raw.trim().length > 0) yield raw; } - if (buf.trim().length > 0) yield buf; } finally { reader.releaseLock(); } diff --git a/src/loop/adapter-google.ts b/src/loop/adapter-google.ts index 4d17d5d374..327c75344c 100644 --- a/src/loop/adapter-google.ts +++ b/src/loop/adapter-google.ts @@ -5,7 +5,7 @@ import { buildVisibilityMarker } from "./core.js"; import { composeStreamFilters, createBiliArtifactFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js"; import { degenerateTurnWarning } from "../degenerate-turn.js"; import { log as loggerLog } from "../logger.js"; -import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData } from "../sse-util.js"; +import { createSseFramer, sseEventData } from "../sse-util.js"; import type { CompressLoopAdapter, @@ -169,6 +169,7 @@ interface GoogleChunk { async function* iterGoogleChunks(stream: ReadableStream): AsyncGenerator { const reader = stream.getReader(); const decoder = new TextDecoder(); + const framer = createSseFramer(); let buf = ""; let mode: "unknown" | "sse" | "array" = "unknown"; let cursor = 0; @@ -200,14 +201,11 @@ async function* iterGoogleChunks(stream: ReadableStream): AsyncGener } } if (mode === "sse") { - buf = normalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.push(buf)) { const json = dataLine(raw); if (json !== null) yield { frame: raw, json }; } + buf = ""; continue; } for (; cursor < buf.length && !arrayDone; cursor++) { @@ -241,18 +239,10 @@ async function* iterGoogleChunks(stream: ReadableStream): AsyncGener } } if (mode === "sse") { - buf = finalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.finish()) { const json = dataLine(raw); if (json !== null) yield { frame: raw, json }; } - if (buf.trim().length > 0) { - const json = dataLine(buf); - if (json !== null) yield { frame: buf, json }; - } } } finally { reader.releaseLock(); diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index 009c8fbd79..a195352343 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -5,7 +5,7 @@ import { buildVisibilityMarker } from "./core.js"; import { composeStreamFilters, createBiliArtifactFilter, createIdentityStreamFilter, createMarkerLineFilter, createTagEchoFilter } from "./tag-echo-filter.js"; import { degenerateTurnWarning } from "../degenerate-turn.js"; import { log as loggerLog } from "../logger.js"; -import { normalizeSseLineEndings, finalizeSseLineEndings, sseEventData, replaceSseEventData } from "../sse-util.js"; +import { createSseFramer, sseEventData, replaceSseEventData } from "../sse-util.js"; import { hardenOpenaiAssistantContent, systemToUser } from "../util.js"; import type { @@ -87,7 +87,7 @@ interface ToolCallBuffer { async function* iterSseChunks(stream: ReadableStream): AsyncGenerator { const reader = stream.getReader(); const decoder = new TextDecoder(); - let buf = ""; + const framer = createSseFramer(); try { while (true) { let done: boolean; @@ -98,23 +98,13 @@ async function* iterSseChunks(stream: ReadableStream): AsyncGenerato break; } if (done) break; - buf += decoder.decode(value, { stream: true }); - buf = normalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.push(decoder.decode(value, { stream: true }))) { if (raw.trim().length > 0) yield raw; } } - buf = finalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.finish()) { if (raw.trim().length > 0) yield raw; } - if (buf.trim().length > 0) yield buf; } finally { reader.releaseLock(); } diff --git a/src/loop/adapter-responses.ts b/src/loop/adapter-responses.ts index a0decf5065..46447fe5a6 100644 --- a/src/loop/adapter-responses.ts +++ b/src/loop/adapter-responses.ts @@ -8,7 +8,7 @@ import { hashId, strippedResponseIdWarning } from "../util.js"; import { composeStreamFilters, createBiliArtifactFilter, createIdentityStreamFilter, createMarkerLineFilter, createTagEchoFilter, stripResponsesText, containsBiliInternalText, containsMarkerLineText, containsRenderTagText, containsToolCallEmissionText, ACP_NAME_ALT, type TagEchoFilter } from "./tag-echo-filter.js"; import { degenerateTurnWarning } from "../degenerate-turn.js"; import { log as loggerLog } from "../logger.js"; -import { normalizeSseLineEndings, finalizeSseLineEndings } from "../sse-util.js"; +import { createSseFramer } from "../sse-util.js"; import { extractResponsesTextTriggers, PROXY_TOOL_NAMES } from "../compress-tool.js"; import type { BiliMessage } from "acp-kernel/wire"; import type { @@ -182,7 +182,7 @@ function rebuildResponsesEvent(type: string, obj: Record): Buff async function* iterSseEvents(stream: ReadableStream): AsyncGenerator { const reader = stream.getReader(); const decoder = new TextDecoder(); - let buf = ""; + const framer = createSseFramer(); try { while (true) { let done: boolean; @@ -193,23 +193,13 @@ async function* iterSseEvents(stream: ReadableStream): AsyncGenerato break; } if (done) break; - buf += decoder.decode(value, { stream: true }); - buf = normalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.push(decoder.decode(value, { stream: true }))) { if (raw.trim().length > 0) yield raw; } } - buf = finalizeSseLineEndings(buf); - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - const raw = buf.slice(0, idx); - buf = buf.slice(idx + 2); + for (const raw of framer.finish()) { if (raw.trim().length > 0) yield raw; } - if (buf.trim().length > 0) yield buf; } finally { reader.releaseLock(); } diff --git a/src/sse-util.ts b/src/sse-util.ts index 9aa7f03a7f..0d7261fa1e 100644 --- a/src/sse-util.ts +++ b/src/sse-util.ts @@ -31,6 +31,56 @@ export function finalizeSseLineEndings(buf: string): string { return buf.replace(/\r\n|\r/g, "\n"); } +/** Frame newly decoded chunks without repeatedly scanning an unfinished + * event. Normalize only the new chunk plus a possible split CR, and join + * incomplete line fragments once when that line ends. Frames preserve the + * existing LF-normalized `split("\n\n")` semantics, including EOF residuals. */ +export function createSseFramer(): { + push(chunk: string): Generator; + finish(): Generator; +} { + let pendingCr = ""; + let lineParts: string[] = []; + let frameParts: string[] = []; + + function* push(chunk: string): Generator { + let text = normalizeSseLineEndings(pendingCr + chunk); + pendingCr = text.endsWith("\r") ? "\r" : ""; + if (pendingCr) text = text.slice(0, -1); + let start = 0; + let end: number; + while ((end = text.indexOf("\n", start)) !== -1) { + if (end > start) lineParts.push(text.slice(start, end)); + if (lineParts.length === 0 && frameParts.length > 0) { + // The preceding line's LF and this blank line's LF are the + // separator; neither belongs to the returned raw frame. + frameParts.pop(); + const frame = frameParts.join(""); + frameParts = []; + yield frame; + } else { + if (lineParts.length > 0) frameParts.push(lineParts.join("")); + frameParts.push("\n"); + } + lineParts = []; + start = end + 1; + } + if (start < text.length) lineParts.push(text.slice(start)); + } + + function* finish(): Generator { + // A final lone CR is one terminator. Resolving it with an LF uses + // the same split-CRLF path and contributes exactly one normalized LF. + if (pendingCr) yield* push("\n"); + const tail = [...frameParts, ...lineParts].join(""); + frameParts = []; + lineParts = []; + if (tail.length > 0) yield tail; + } + + return { push, finish }; +} + /** Data fields of one already-framed, LF-normalized SSE event form ONE * payload, joined by LF. Remove only the optional first space after `:`; * indentation and empty fields are part of the payload. */ diff --git a/tests/loop-sse-framing-scan.test.ts b/tests/loop-sse-framing-scan.test.ts new file mode 100644 index 0000000000..98b3725691 --- /dev/null +++ b/tests/loop-sse-framing-scan.test.ts @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createGoogleAdapter } from "../src/loop/adapter-google.ts"; +import { createSseFramer, finalizeSseLineEndings } from "../src/sse-util.ts"; + +function chunks(wire: Buffer, chunkSize: number): ReadableStream { + let offset = 0; + return new ReadableStream({ pull(controller) { + if (offset >= wire.length) { controller.close(); return; } + controller.enqueue(wire.subarray(offset, offset + chunkSize)); + offset += chunkSize; + } }); +} + +function googleFrame(payload: string): string { + return `data: ${JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "Generated image." }, { inlineData: { mimeType: "image/png", data: payload } }] } }] })}\n\n`; +} + +async function googleOutput(wire: Buffer, chunkSize: number): Promise { + const adapter = createGoogleAdapter({}, undefined, undefined, "gemini-test"); + const parts: Buffer[] = []; + for await (const event of adapter.parseStream(chunks(wire, chunkSize), 1)) { + if (event.kind === "meta") parts.push(event.chunk); + else if ("raw" in event && event.raw) parts.push(event.raw); + } + return Buffer.concat(parts).toString("utf8"); +} + +test("Google loop framing scans newly received bytes, not the accumulated image prefix", async (t) => { + const frame = googleFrame("A".repeat(1024 * 1024)); + const wire = Buffer.from(frame); + const indexOf = String.prototype.indexOf; + let searchedChars = 0; + t.mock.method(String.prototype, "indexOf", function (this: string, needle: string, position?: number): number { + const value = String(this); + const result = indexOf.call(value, needle, position); + if (needle === "\r" || needle === "\n\n" || needle === "\n") { + const start = Math.max(0, position ?? 0); + searchedChars += Math.max(0, (result < 0 ? value.length : result + needle.length) - start); + } + return result; + }); + assert.equal(await googleOutput(wire, 4 * 1024), frame, "the complete image/text frame stays byte-exact"); + t.diagnostic(`SSE searches examined ${searchedChars} characters for ${wire.length} received bytes`); + assert.ok(searchedChars <= wire.length * 8, `SSE searches examined ${searchedChars} characters for ${wire.length} received bytes`); +}); + +for (const ending of ["\n", "\r\n", "\r"]) { + test(`Google loop framing retains Unicode across one-byte ${JSON.stringify(ending)} splits`, async () => { + const normalized = googleFrame("你好🔥"); + const wire = Buffer.from(normalized.replaceAll("\n", ending)); + assert.equal(await googleOutput(wire, 1), normalized); + }); +} + +function frameChunks(parts: string[]): string[] { + const framer = createSseFramer(); + return [...parts.flatMap(part => [...framer.push(part)]), ...framer.finish()]; +} + +function wholeFrames(text: string): string[] { + const frames = finalizeSseLineEndings(text).split("\n\n"); + if (frames.at(-1) === "") frames.pop(); + return frames; +} + +test("incremental framing preserves complete-field whitespace and EOF residuals at every split", () => { + const inputs = [ + "", "\n", "\r", "\r\n", "\n\n\n", "\n\ndata: tail", "\ndata: tail\n", + ":comment\r\nid: 1\r\nevent: update\r\ndata: {\r\ndata: \"text\": \"你好🔥\"\r\ndata: }\r\n\r\n", + "data\ndata: indented\ndata:\n\ndata: residual\n", + "event: partial\rdata: {\"text\":\"tail\"}\r", + ]; + for (const input of inputs) { + const expected = wholeFrames(input); + for (let cut = 0; cut <= input.length; cut++) { + assert.deepEqual(frameChunks([input.slice(0, cut), input.slice(cut)]), expected, `split ${cut}: ${JSON.stringify(input)}`); + } + assert.deepEqual(frameChunks(input.split("")), expected, `one-character chunks: ${JSON.stringify(input)}`); + } +}); + +test("all short CR/LF combinations match the previous whole-buffer framing", () => { + const alphabet = ["a", "\r", "\n"]; + let values = [""]; + for (let length = 0; length <= 6; length++) { + for (const value of values) { + const expected = wholeFrames(value); + for (let cut = 0; cut <= value.length; cut++) { + assert.deepEqual(frameChunks([value.slice(0, cut), value.slice(cut)]), expected); + } + } + values = values.flatMap(value => alphabet.map(char => value + char)); + } +});