Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions scripts/bench-terminal-sse.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Manual benchmark: node --import tsx scripts/bench-terminal-sse.ts [MiB] [chunk-KiB]
// Reports timings; no machine-dependent timing threshold runs in CI.
import { performance } from "node:perf_hooks";
import { observeResponsesTerminalState } from "../src/stream-terminal.ts";

const mib = Number(process.argv[2] ?? 8);
const chunkKiB = Number(process.argv[3] ?? 32);
if (!Number.isFinite(mib) || mib <= 0 || !Number.isFinite(chunkKiB) || chunkKiB <= 0) {
throw new Error("MiB and chunk-KiB must be positive numbers");
}
const size = Math.max(1, Math.floor(chunkKiB * 1024));
const bytes = new TextEncoder().encode(`event: response.completed\r\ndata: ${JSON.stringify({
type: "response.completed", response: { output: [{ type: "compaction", encrypted_content: "x".repeat(Math.floor(mib * 1024 * 1024)) }] },
})}\r\n\r\n`);
const elapsed: number[] = [];
for (let round = 0; round < 4; round++) {
let offset = 0;
const source = new ReadableStream<Uint8Array>({
pull(controller) {
if (offset === bytes.length) return controller.close();
controller.enqueue(bytes.subarray(offset, offset + size));
offset = Math.min(bytes.length, offset + size);
},
});
const start = performance.now();
if (await observeResponsesTerminalState(source, true) !== "completed") throw new Error("terminal verdict changed");
const ms = performance.now() - start;
if (round > 0) elapsed.push(ms);
}
elapsed.sort((a, b) => a - b);
console.log(JSON.stringify({ mib, chunkKiB, samplesMs: elapsed, medianMs: elapsed[1] }));
43 changes: 27 additions & 16 deletions src/stream-terminal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
// never happened discards valid blocks (#249). "unknown" is the safe default
// (no rebase; the next request self-heals via kernel message-id deactivation).

import { finalizeSseLineEndings, normalizeSseLineEndings } from "./sse-util.js";

type TerminalState = "completed" | "failed" | "unknown";

const SUCCEEDED = "response.completed";
Expand Down Expand Up @@ -40,30 +38,43 @@ function terminalFromSseFrame(frame: string): TerminalState | null {
async function observeSseTerminalState(stream: ReadableStream<Uint8Array>): Promise<TerminalState> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buf = "";
let lineParts: string[] = [];
let frameLines: string[] = [];
let skipLeadingLf = false;
let state: TerminalState = "unknown";
const consumeFrames = () => {
let boundary: number;
while ((boundary = buf.indexOf("\n\n")) >= 0) {
const frame = buf.slice(0, boundary);
buf = buf.slice(boundary + 2);
const next = terminalFromSseFrame(frame);
if (next !== null) {
state = next;
break;
// A response.completed event can contain the entire compacted response.
// Scan each decoded chunk once and join each line/frame once, rather than
// re-normalizing and searching an ever-growing multi-MiB prefix per chunk.
const consumeText = (text: string) => {
let start = 0;
for (let index = 0; index < text.length; index++) {
const code = text.charCodeAt(index);
if (skipLeadingLf) {
skipLeadingLf = false;
if (code === 10) { start = index + 1; continue; }
}
if (code !== 10 && code !== 13) continue;
lineParts.push(text.slice(start, index));
const line = lineParts.join("");
lineParts = [];
if (line.length === 0) {
const next = terminalFromSseFrame(frameLines.join("\n"));
frameLines = [];
if (next !== null) { state = next; return; }
} else frameLines.push(line);
skipLeadingLf = code === 13;
start = index + 1;
}
if (start < text.length) lineParts.push(text.slice(start));
};
try {
for (;;) {
const { done, value } = await reader.read();
if (done) {
buf = finalizeSseLineEndings(buf + decoder.decode());
consumeFrames();
consumeText(decoder.decode());
break;
}
buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true }));
consumeFrames();
consumeText(decoder.decode(value, { stream: true }));
if (state !== "unknown") {
// Release this tee branch; the client branch keeps draining the source.
await reader.cancel().catch(() => {});
Expand Down
43 changes: 43 additions & 0 deletions tests/terminal-sse-large-frames.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { observeResponsesTerminalState } from "../src/stream-terminal.ts";

function chunked(text: string, size: number): ReadableStream<Uint8Array> {
const bytes = new TextEncoder().encode(text);
let offset = 0;
return new ReadableStream<Uint8Array>({
pull(controller) {
if (offset === bytes.length) return controller.close();
controller.enqueue(bytes.subarray(offset, offset + size));
offset = Math.min(bytes.length, offset + size);
},
});
}

for (const eol of ["\n", "\r\n", "\r"]) {
test(`large native compaction terminal frames keep ${JSON.stringify(eol)} boundaries`, async () => {
const payload = JSON.stringify({ type: "response.completed", response: {
output: [{ type: "compaction", encrypted_content: "x".repeat(2 * 1024 * 1024) }],
} });
const wire = ["event: response.created", 'data: {"type":"response.created"}', "",
"event: response.completed", `data: ${payload}`, "", ""].join(eol);
assert.equal(await observeResponsesTerminalState(chunked(wire, 8191), true), "completed");
});
}

test("a long invalid frame cannot consume the following terminal frame", async () => {
const wire = `data: ${"x".repeat(2 * 1024 * 1024)}\r\n\r\n`
+ 'data: {"type":"response.failed"}\r\n\r\n';
assert.equal(await observeResponsesTerminalState(chunked(wire, 8191), true), "failed");
});

test("chunked Unicode and multiline JSON retain the first terminal verdict", async () => {
const wire = ['\ufeffevent: response.completed', 'data: {"text":"你好🙂",',
'data: "type":"response.failed"}', "", 'data: {"type":"response.completed"}', "", ""].join("\r\n");
assert.equal(await observeResponsesTerminalState(chunked(wire, 1), true), "failed");
});

test("an unterminated large terminal event stays unknown", async () => {
const wire = `data: ${JSON.stringify({ type: "response.completed", text: "x".repeat(2 * 1024 * 1024) })}\n`;
assert.equal(await observeResponsesTerminalState(chunked(wire, 8191), true), "unknown");
});
Loading