From 7e1a4b954fa983362d512755f21789b51b7cdc02 Mon Sep 17 00:00:00 2001 From: Afloat Date: Thu, 19 Nov 2026 00:13:34 +0800 Subject: [PATCH] fix: release consumed Google array stream prefixes --- scripts/bench-google-array-stream.mjs | 98 +++++++++++++ src/loop/adapter-google.ts | 13 ++ tests/google-array-buffer.test.ts | 197 ++++++++++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 scripts/bench-google-array-stream.mjs create mode 100644 tests/google-array-buffer.test.ts diff --git a/scripts/bench-google-array-stream.mjs b/scripts/bench-google-array-stream.mjs new file mode 100644 index 0000000000..e587607985 --- /dev/null +++ b/scripts/bench-google-array-stream.mjs @@ -0,0 +1,98 @@ +// Compare the same Google JSON-array workload in two checkouts, alternating run order. +// node --expose-gc --import tsx scripts/bench-google-array-stream.mjs --baseline-root /path/to/base +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const argv = process.argv.slice(2); +const option = (name, fallback) => argv.includes(name) ? argv[argv.indexOf(name) + 1] : fallback; +const root = path.resolve(option("--root", path.join(path.dirname(fileURLToPath(import.meta.url)), ".."))); +const baselineRoot = option("--baseline-root"); +const counts = option("--counts", "1024,2048,4096").split(",").map(Number); +const pairs = Number(option("--pairs", "5")); +assert.ok(counts.every(count => Number.isSafeInteger(count) && count > 0)); +assert.ok(Number.isSafeInteger(pairs) && pairs > 0); +assert.equal(typeof global.gc, "function", "run Node with --expose-gc for comparable allocation profiles"); + +async function source(label, directory) { + const file = realpathSync(path.join(directory, "src/loop/adapter-google.ts")); + const sourceSha256 = createHash("sha256").update(readFileSync(file)).digest("hex"); + const { createGoogleAdapter } = await import(pathToFileURL(file).href); + return { label, file, sourceSha256, createGoogleAdapter }; +} + +const sources = baselineRoot + ? [await source("baseline", path.resolve(baselineRoot)), await source("candidate", root)] + : [await source("current", root)]; +if (sources.length === 2) assert.notEqual(sources[0].sourceSha256, sources[1].sourceSha256, "baseline and candidate must contain different adapter source"); +const json = JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "ordinary text ".repeat(72) }] }, index: 0 }] }); +const terminal = JSON.stringify({ candidates: [{ finishReason: "STOP", index: 0 }] }); + +async function measure(source, count, mode, profile = false, trailingChunks = 0) { + global.gc(); + const heapStart = process.memoryUsage().heapUsed; + let sent = 0, wireBytes = 0, outputBytes = 0, textEvents = 0, doneEvents = 0, retainedHeapBytes; + const digest = createHash("sha256"); + const stream = new ReadableStream({ pull(controller) { + if (sent > count + trailingChunks) return controller.close(); + let text; + if (sent > count) { + text = " ".repeat(1024); + if (profile && sent - count === Math.floor(trailingChunks * 0.75)) { + global.gc(); + retainedHeapBytes = process.memoryUsage().heapUsed - heapStart; + } + } else { + const value = sent === count ? terminal : json; + text = mode === "sse" ? `data: ${value}\n\n` : `${sent === 0 ? "[" : ","}${value}${sent === count ? "]" : ""}`; + } + const bytes = Buffer.from(text); + wireBytes += bytes.byteLength; + controller.enqueue(bytes); + sent++; + } }); + const start = performance.now(); + for await (const event of source.createGoogleAdapter({}).parseStream(stream, 1)) { + const raw = "raw" in event && event.raw ? event.raw : event.kind === "meta" ? event.chunk : undefined; + if (raw) { outputBytes += raw.byteLength; digest.update(raw); } + if (event.kind === "done") doneEvents++; + if (event.kind === "text" && ++textEvents === Math.floor(count * 0.75) && profile && trailingChunks === 0) { + global.gc(); + retainedHeapBytes = process.memoryUsage().heapUsed - heapStart; + } + } + const elapsedMs = performance.now() - start; + assert.equal(textEvents, count); + assert.equal(doneEvents, 1); + return { source: source.label, mode, count, trailingChunks, wireBytes, outputBytes, digest: digest.digest("hex"), elapsedMs, ...(profile ? { retainedHeapBytes } : {}) }; +} + +const results = [], profiles = [], controls = []; +const expected = new Map(); +function verify(row) { + const key = row.count; + const value = `${row.outputBytes}:${row.digest}`; + if (expected.has(key)) assert.equal(value, expected.get(key), "forwarded bytes must match for every source and framing"); + else expected.set(key, value); + return row; +} +for (const item of sources) await measure(item, 256, "array"); +for (const count of counts) { + for (let pair = 0; pair < pairs; pair++) { + const order = pair % 2 === 0 ? sources : [...sources].reverse(); + for (const item of order) results.push({ pair, ...verify(await measure(item, count, "array")) }); + } + for (const item of sources) { + profiles.push(verify(await measure(item, count, "array", true))); + controls.push(verify(await measure(item, count, "sse"))); + } +} +for (const item of sources) profiles.push(verify(await measure(item, 16, "array", true, 4096))); +const medians = sources.flatMap(item => counts.map(count => { + const times = results.filter(row => row.source === item.label && row.count === count).map(row => row.elapsedMs).sort((a, b) => a - b); + return { source: item.label, count, medianMs: times[Math.floor(times.length / 2)] }; +})); +console.log(JSON.stringify({ node: process.version, platform: process.platform, architecture: process.arch, sources: sources.map(({ createGoogleAdapter: _factory, ...rest }) => rest), pairs, medians, results, profiles, controls }, null, 2)); diff --git a/src/loop/adapter-google.ts b/src/loop/adapter-google.ts index 9e9fe8c6dc..9f8f39a3e0 100644 --- a/src/loop/adapter-google.ts +++ b/src/loop/adapter-google.ts @@ -237,6 +237,19 @@ async function* iterGoogleChunks(stream: ReadableStream): AsyncGener } } } + // Keep only the unfinished element across reads. Retaining completed + // prefixes can repeatedly copy the history during concatenation/indexing. + if (arrayDone) { + buf = ""; + cursor = 0; + } else { + const consumed = started ? elementStart : cursor; + if (consumed > 0) { + buf = buf.slice(consumed); + cursor -= consumed; + if (started) elementStart = 0; + } + } } if (mode === "sse") { for (const raw of framer.finish()) { diff --git a/tests/google-array-buffer.test.ts b/tests/google-array-buffer.test.ts new file mode 100644 index 0000000000..3a0f58e73d --- /dev/null +++ b/tests/google-array-buffer.test.ts @@ -0,0 +1,197 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { setImmediate as nextTurn } from "node:timers/promises"; +import { defaultConfig } from "acp-kernel"; +import { createGoogleAdapter } from "../src/loop/adapter-google.ts"; +import type { ParsedStreamEvent } from "../src/loop/core.ts"; +import { sseEventData } from "../src/sse-util.ts"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { closeServers } from "./server-close-guard.ts"; + +function candidate(parts: Array>, finishReason?: string): Record { + return { candidates: [{ index: 0, content: { role: "model", parts }, ...(finishReason ? { finishReason } : {}) }] }; +} + +const first = candidate([{ text: "First café 🛰️. " }]); +const signed = { thought: true, text: 'Nested delimiters: [{"quoted":"\\\\ and \\\""}] 🌏.', thoughtSignature: "c2lnbmVk" }; +const middle = candidate([signed, { text: "Second answer. " }]); +const args = { query: 'escaped \\" quote and } ] [ {', nested: [{ values: [1, true, null, "多字节"] }], empty: {} }; +const call = candidate([{ functionCall: { id: "lookup-1", name: "lookup", args }, thoughtSignature: "dG9vbC1zaWc=" }]); +const last = { ...candidate([], "STOP"), usageMetadata: { promptTokenCount: 20, candidatesTokenCount: 5, thoughtsTokenCount: 3 } }; +const frames = [first, middle, call, last]; + +function arrayStream(wire: Buffer, chunkSize: number | number[], failAfterData = false): ReadableStream { + let offset = 0; + let reads = 0; + return new ReadableStream({ pull(controller) { + if (offset === wire.length) { + if (failAfterData) controller.error(new Error("Synthetic stream interruption.")); + else controller.close(); + return; + } + const size = typeof chunkSize === "number" ? chunkSize : chunkSize[reads++ % chunkSize.length]!; + controller.enqueue(wire.subarray(offset, offset + size)); + offset = Math.min(offset + size, wire.length); + } }); +} + +async function collect(wire: string, chunkSize: number | number[], failAfterData = false): Promise { + const events: ParsedStreamEvent[] = []; + for await (const event of createGoogleAdapter({}).parseStream(arrayStream(Buffer.from(wire), chunkSize, failAfterData), 1)) events.push(event); + return events; +} + +test("Google array keeps a partial escaped string after a completed element in the same read", async () => { + const wire = JSON.stringify(frames); + const middleStart = wire.indexOf(JSON.stringify(middle)); + const splitAfterEscape = wire.indexOf("\\", middleStart) + 1; + assert.ok(splitAfterEscape > middleStart); + const firstReadBytes = Buffer.byteLength(wire.slice(0, splitAfterEscape)); + const events = await collect(wire, [firstReadBytes, 1, 2, 7]); + const control = await collect(wire, Buffer.byteLength(wire)); + assert.deepEqual(semanticEvents(events), semanticEvents(control)); + assert.deepEqual(forwarded(events), forwarded(control)); +}); + +test("Google array ignores later whitespace chunks after the top-level array closes", async () => { + const wire = JSON.stringify([first, last]); + const control = await collect(wire, 73); + const events = await collect(wire + " \r\n\t".repeat(8192), [Buffer.byteLength(wire), 17]); + assert.deepEqual(semanticEvents(events), semanticEvents(control)); + assert.deepEqual(forwarded(events), forwarded(control)); +}); + +test("Google array preserves several elements in one read while the consumer yields", async () => { + const wire = JSON.stringify(frames); + const events: ParsedStreamEvent[] = []; + for await (const event of createGoogleAdapter({}).parseStream(arrayStream(Buffer.from(wire), Buffer.byteLength(wire)), 1)) { + events.push(event); + await nextTurn(); + } + assert.deepEqual(semanticEvents(events), semanticEvents(await collect(wire, 23))); +}); + +function semanticEvents(events: ParsedStreamEvent[]): Array> { + return events.map(event => { + const { raw: _raw, chunk, ...rest } = event as ParsedStreamEvent & { raw?: Buffer; chunk?: Buffer }; + return chunk ? { ...rest, frame: JSON.parse(sseEventData(chunk.toString())!) as unknown } : rest; + }); +} + +function forwarded(events: ParsedStreamEvent[]): Array> { + return events.flatMap(event => { + const raw = "raw" in event && event.raw ? event.raw : event.kind === "meta" ? event.chunk : undefined; + return raw ? [JSON.parse(sseEventData(raw.toString())!) as Record] : []; + }); +} + +for (const pretty of [false, true]) { + for (const chunkSize of [1, 2, 7, 31, 113, 4096]) { + test(`Google array framing preserves nested values and split UTF-8 (${pretty ? "pretty" : "compact"}, ${chunkSize}-byte chunks)`, async () => { + const control = await collect(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(""), 43); + const wire = `\ufeff \n${JSON.stringify(frames, null, pretty ? 2 : undefined)}\n`; + const events = await collect(wire, chunkSize); + assert.deepEqual(semanticEvents(events), semanticEvents(control)); + assert.deepEqual(forwarded(events), forwarded(control)); + const tool = events.find(event => event.kind === "tool_call"); + assert.ok(tool && tool.kind === "tool_call"); + assert.deepEqual(JSON.parse(tool.arguments), args); + assert.equal(tool.signature, "dG9vbC1zaWc="); + assert.deepEqual(events.filter(event => event.kind === "reasoning").map(event => event.delta), [signed.text]); + }); + } +} + +for (const interrupted of [false, true]) { + test(`Google array retains complete-prefix output before an ${interrupted ? "interrupted" : "incomplete"} element`, async () => { + const wire = `[${JSON.stringify(first)},${JSON.stringify(middle)},{"candidates":[{"content":{"parts":[{"text":"unfinished`; + const events = await collect(wire, 19, interrupted); + assert.deepEqual(forwarded(events), [first, middle]); + assert.equal(events.filter(event => event.kind === "done").length, 0); + assert.ok(!JSON.stringify(events).includes("unfinished")); + }); +} + +test("Google array emits an element before the next network read can finish", { timeout: 5000 }, async t => { + let resume!: () => void; + const resumed = new Promise(resolve => { resume = resolve; }); + t.signal.addEventListener("abort", resume, { once: true }); + let pulls = 0; + const stream = new ReadableStream({ async pull(controller) { + if (pulls++ === 0) controller.enqueue(Buffer.from(`[${JSON.stringify(first)},`)); + else { + await resumed; + controller.enqueue(Buffer.from(`${JSON.stringify(last)}]`)); + controller.close(); + } + } }); + const iterator = createGoogleAdapter({}).parseStream(stream, 1); + try { + const event = await iterator.next(); + assert.equal(event.value?.kind, "text"); + if (event.value?.kind === "text") assert.equal(event.value.delta, "First café 🛰️. "); + } finally { + resume(); + for await (const _event of iterator) { /* finish the remaining response */ } + } +}); + +test("Google array response reaches the actual HTTP compression loop", { timeout: 15000 }, async () => { + let requests = 0; + let toolInjected = false; + const output = [first, middle, candidate([{ text: "Last answer." }]), last]; + const upstream = http.createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + requests++; + const body = JSON.parse(Buffer.concat(chunks).toString()) as Record; + toolInjected = JSON.stringify(body.tools).includes("compress"); + res.writeHead(200, { "content-type": "application/json" }); + for (let i = 0; i < output.length; i++) { + res.write(`${i === 0 ? "[" : ","}${JSON.stringify(output[i])}`); + await nextTurn(); + } + res.end("]"); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamAddress = upstream.address(); + assert.ok(upstreamAddress && typeof upstreamAddress === "object"); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const opts: ProxyOptions = { + port: 0, host: "127.0.0.1", upstream: `http://127.0.0.1:${upstreamAddress.port}`, + routes: {}, proxy: "", proxyMode: "direct", proxySource: "direct", + modelContextLimit: 200000, kernelConfig: defaultConfig(200000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", + log: false, debug: false, passthrough: false, chainContentDetection: false, + autoUpdate: false, mitm: { enabled: false, domains: [] }, + }; + let proxy: http.Server | undefined; + try { + proxy = await startServer(opts); + if (!proxy.listening) await once(proxy, "listening"); + const address = proxy.address(); + assert.ok(address && typeof address === "object"); + const response = await fetch(`http://127.0.0.1:${address.port}/v1beta/models/gemini-test:streamGenerateContent`, { + method: "POST", headers: { "content-type": "application/json", "x-acp-session": "google-array-buffer" }, + body: JSON.stringify({ contents: [{ role: "user", parts: [{ text: "Explain the result." }] }] }), + }); + assert.equal(response.status, 200); + const wire = await response.text(); + const received = wire.split("\n\n").map(sseEventData).filter((value): value is string => value !== null && value !== undefined).map(value => JSON.parse(value) as { candidates?: Array<{ content?: { parts?: unknown[] }; finishReason?: string }> }); + const parts = received.flatMap(frame => frame.candidates?.[0]?.content?.parts ?? []); + assert.deepEqual(parts, [{ text: "First café 🛰️. " }, signed, { text: "Second answer. " }, { text: "Last answer." }]); + assert.ok(received.some(frame => frame.candidates?.[0]?.finishReason === "STOP")); + assert.equal(requests, 1); + assert.equal(toolInjected, true); + } finally { + await closeServers(proxy ? [proxy, upstream] : [upstream]); + } +});