From 815334f041726f417e6e214eb9985884c1af2111 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Fri, 4 Sep 2026 06:51:28 +0800 Subject: [PATCH 1/2] feat(history-import): benchmark ConversationView open/scroll/stream against the Mirror baseline (phase 1b-C) `bench:open` now defaults to the deterministic synthetic replay at `--scale=1,10` (`--fixture` keeps taking a desensitized capture) and measures the view next to the full-Mirror baseline: `view.open` (import + view + tail hydrate -> one renderable row per turn), `view.readAll` (the `doc.history` bridge's first read), `view.scroll` (a 30-turn `ensureRange` window advancing 20 times, p99), `view.stream` (100 text deltas into the tail turn with the view attached, p99) and `view.append`. View tasks run before the baseline so the Mirror's garbage is not charged to them, and the summary prints the phase 1b acceptance checks (open <= 50 ms at x10, stream p99 <= 4 ms). Two view fixes the benchmark surfaced: - `rebuildIndex` / `evict` resolved positions with `ids.indexOf` per turn, O(n^2) per structural change (an append at 2,400 turns took seconds); they use a position map now. - `itemCount` is read for assistant turns only and `planCount` only when a plan exists, which trims two of the five wasm calls per turn at open. Model: claude-fable-5-1 --- .../src/lib/conversation-view/AGENTS.md | 8 +- .../conversation-view/conversation-view.ts | 31 +- .../tests/conversation-view.test.ts | 3 +- packages/history-import/AGENTS.md | 14 + .../benchmarks/open-conversation.bench.ts | 297 +++++++++++++++--- 5 files changed, 304 insertions(+), 49 deletions(-) diff --git a/packages/components/src/lib/conversation-view/AGENTS.md b/packages/components/src/lib/conversation-view/AGENTS.md index 7001e274b..b7b3d2859 100644 --- a/packages/components/src/lib/conversation-view/AGENTS.md +++ b/packages/components/src/lib/conversation-view/AGENTS.md @@ -11,9 +11,11 @@ long conversation never materializes every turn through a Mirror. ## Contracts - `index(i)` is always loaded and comes from the turn map's shallow value plus - `summary`, `itemCount` and `planCount`. Add a field to `TURN_INDEX_FIELDS` - only when a reader that must stay O(1) needs it; every field costs one - shallow read per turn at open. + `summary`, `itemCount` (assistant turns only) and `planCount` (when a plan + exists). Open cost is two wasm calls per turn plus one per assistant turn + (~20 µs, ~50 ms for 2,400 turns); add a field to `TURN_INDEX_FIELDS` only + when a reader that must stay O(1) needs it, and never one that needs + another container read per turn. - `turn(i)` is synchronous only for hydrated turns. Hydration is per-turn `toJSON()`; the LRU never evicts the tail (`tailKeep`), a `retain()`ed range, or the range an `ensureRange()` call just asked for. A caller that diff --git a/packages/components/src/lib/conversation-view/conversation-view.ts b/packages/components/src/lib/conversation-view/conversation-view.ts index 53f9674c7..2e54805cb 100644 --- a/packages/components/src/lib/conversation-view/conversation-view.ts +++ b/packages/components/src/lib/conversation-view/conversation-view.ts @@ -20,6 +20,11 @@ export const TURN_INDEX_FIELDS = [ export type TurnIndexRow = Pick & { summary?: TurnSummary; + /** + * Read for assistant turns only: the empty-turn rule and the height + * estimate need it there, and each read is one more wasm call per turn + * at open. User turns always carry their prompt. + */ itemCount?: number; planCount?: number; }; @@ -151,17 +156,25 @@ export function createConversationViewFromDoc( ? (doc.getContainerById(summary as never)?.toJSON() as TurnSummary | undefined) : summary; } - const itemCount = containerLength(doc, shallow.items); - if (itemCount !== undefined) row.itemCount = itemCount; - const planCount = containerLength(doc, shallow.plan); - if (planCount !== undefined) row.planCount = planCount; + if (shallow.role === 'assistant') { + const itemCount = containerLength(doc, shallow.items); + if (itemCount !== undefined) row.itemCount = itemCount; + } + if (shallow.plan !== undefined) { + const planCount = containerLength(doc, shallow.plan); + if (planCount !== undefined) row.planCount = planCount; + } return row as TurnIndexRow; }; + /** Position of every container id in `ids`; rebuilt with the index. */ + let positionByCid = new Map(); + const rebuildIndex = () => { const shallow = list.getShallowValue() as unknown[]; const nextIds: (string | null)[] = new Array(shallow.length); const nextRows: (TurnIndexRow | undefined)[] = new Array(shallow.length); + const nextPositionByCid = new Map(); positionById.clear(); for (let i = 0; i < shallow.length; i += 1) { const cid = shallow[i]; @@ -171,11 +184,12 @@ export function createConversationViewFromDoc( continue; } nextIds[i] = cid; + nextPositionByCid.set(cid, i); // Reuse the previous row when the container did not move so a structural // change costs one shallow read for the list, not one per turn. - const previousPosition = ids.indexOf(cid); + const previousPosition = positionByCid.get(cid); const row = - previousPosition >= 0 && previousPosition === i + previousPosition === i ? (indexRows[previousPosition] ?? readIndexRow(cid)) : readIndexRow(cid); nextRows[i] = row; @@ -194,6 +208,7 @@ export function createConversationViewFromDoc( } ids = nextIds; indexRows = nextRows; + positionByCid = nextPositionByCid; }; const materialize = (i: number): SessionHistory | undefined => { @@ -230,8 +245,8 @@ export function createConversationViewFromDoc( if (hydrated.size <= maxHydrated) return; const candidates: { cid: string; lastUsed: number }[] = []; for (const [cid, entry] of hydrated) { - const position = ids.indexOf(cid); - if (position >= 0 && isProtected(position, keep)) continue; + const position = positionByCid.get(cid); + if (position !== undefined && isProtected(position, keep)) continue; candidates.push({ cid, lastUsed: entry.lastUsed }); } candidates.sort((left, right) => left.lastUsed - right.lastUsed); diff --git a/packages/components/tests/conversation-view.test.ts b/packages/components/tests/conversation-view.test.ts index 3a506527e..4e18e92f1 100644 --- a/packages/components/tests/conversation-view.test.ts +++ b/packages/components/tests/conversation-view.test.ts @@ -96,7 +96,8 @@ describe('ConversationView', () => { const doc = docWithTurns(60); const view = createConversationViewFromDoc(doc, { sessionId, tailKeep: 5, maxHydrated: 10 }); expect(view.turnCount).toBe(60); - expect(view.index(0)).toMatchObject({ id: 'u0', role: 'user', itemCount: 1 }); + expect(view.index(0)).toMatchObject({ id: 'u0', role: 'user' }); + expect(view.index(0)?.itemCount).toBeUndefined(); expect(view.index(1)).toMatchObject({ id: 'a1', role: 'assistant', itemCount: 3 }); expect(view.indexOf('a59')).toBe(59); expect(view.isHydrated(59)).toBe(true); diff --git a/packages/history-import/AGENTS.md b/packages/history-import/AGENTS.md index 5c5e491f9..bea8bf2cd 100644 --- a/packages/history-import/AGENTS.md +++ b/packages/history-import/AGENTS.md @@ -67,3 +67,17 @@ Baseline on an M-series laptop, real ~170-turn / ~5k-item session doc: phase 2 i ~3ms `LoroDoc.import` plus ~445ms of Mirror construction, and phase 2 dominates. Mirror init walks every container (one `LoroMap` per message item plus a `LoroText` per text item), so its cost tracks container count, not bytes. + +`pnpm --filter @lody/history-import bench:open` isolates that open cost and +compares it with the client's `ConversationView` +(`packages/components/src/lib/conversation-view`, imported by relative path: +the view depends only on loro-crdt and `@lody/shared` types, so it is the one +piece of client code a benchmark here may reach). Default fixture is the +synthetic replay at `--scale=1,10`; `--fixture=` takes a desensitized +capture from `bench:capture` (never committed). Tasks: the Mirror baseline, +`view.open` (import + view + tail hydrate → one row per turn), `view.readAll` +(what a reader still on the `doc.history` bridge pays), `view.scroll` (a +30-turn `ensureRange` window advancing 20 times, p99), `view.stream` (100 text +deltas into the tail turn with the view attached, p99) and `view.append`. It +prints the phase 1b acceptance checks: `view.open` ≤ 50 ms at x10 and +`view.stream` p99 ≤ 4 ms. diff --git a/packages/history-import/benchmarks/open-conversation.bench.ts b/packages/history-import/benchmarks/open-conversation.bench.ts index 03f02b859..55c9bf7e5 100644 --- a/packages/history-import/benchmarks/open-conversation.bench.ts +++ b/packages/history-import/benchmarks/open-conversation.bench.ts @@ -1,23 +1,40 @@ /** * tinybench baseline for opening a long conversation: Loro snapshot -> - * render-ready `SessionHistory[]`. + * render-ready rows, before (full-schema Mirror) and after (ConversationView). * - * The fixture is a desensitized real Lody conversation (see `capture-fixture.ts`), - * replicated `--scale` times to model conversations longer than anything on this - * machine today. Replication rewrites entry ids so the history list keeps unique - * keys, and every replica is a distinct container subtree, so container count — - * the thing this cost actually tracks — scales linearly with `--scale`. + * The fixture is synthetic by default (`./fixture.ts`, deterministic), or a + * desensitized real Lody conversation via `--fixture` (see `capture-fixture.ts`; + * never commit one). Either is replicated `--scale` times to model conversations + * longer than anything on this machine today. Replication rewrites entry ids so + * the history list keeps unique keys, and every replica is a distinct container + * subtree, so container count — the thing the open cost actually tracks — + * scales linearly with `--scale`. * - * pnpm --filter @lody/history-import bench:open -- --fixture=/tmp/fixture.json - * pnpm --filter @lody/history-import bench:open -- --fixture=... --scale=10 --iterations=10 - * pnpm --filter @lody/history-import bench:open -- --fixture=... --scale=1,10,100 + * pnpm --filter @lody/history-import bench:open + * pnpm --filter @lody/history-import bench:open -- --turns=240 --scale=1,10 + * pnpm --filter @lody/history-import bench:open -- --fixture=/tmp/fixture.json --scale=1,10 --iterations=10 * * Tasks per scale: * import LoroDoc.import(snapshot) — decode only * toJSON doc.toJSON() — full bulk materialization in Rust, no ids * getDeepValueWithID same, carrying container ids (what a bulk Mirror needs) - * Mirror new Mirror(...) — what a client pays today + * Mirror import + new Mirror(...) — what a client paid before * getState mirror.getState() — cached read + * view.open import + createConversationViewFromDoc + tail hydrate + * -> one renderable row per turn (placeholder or message). + * The renderer's `buildChatStreamItemsFromView` adds only + * item normalization for the hydrated tail on top of this. + * view.readAll import + view + readAll(): what a reader still on the + * `doc.history` bridge pays on first access (toJSON per turn, + * no Mirror) + * view.scroll ensureRange over a 30-turn window that advances each + * iteration (20 iterations): what a scroll costs + * view.stream one text delta into the tail turn + commit, with the + * view attached (100 iterations): what a streamed token costs + * view.append appendHistoryEntry of one user turn with the view attached + * + * Acceptance from the phase 1b task: view.open <= 50 ms at x10 (~2,400 turns) + * and view.stream p99 <= 4 ms. The summary line prints both checks. */ import { readFileSync } from 'node:fs'; @@ -25,11 +42,28 @@ import os from 'node:os'; import path from 'node:path'; import { sessionDocSchema, type SessionHistoryInput, type SessionId } from '@lody/shared'; -import { LoroDoc } from 'loro-crdt'; +import { LoroDoc, LoroList, LoroMap, LoroText } from 'loro-crdt'; import { Mirror } from 'loro-mirror'; import { Bench } from 'tinybench'; +// Benchmarks are the one consumer allowed to reach into the client's read +// model: the view is the thing being measured, and it depends on nothing but +// loro-crdt and @lody/shared types. +import { + createConversationViewFromDoc, + type ConversationView, +} from '../../components/src/lib/conversation-view/conversation-view'; +import { appendHistoryEntry } from '../../components/src/lib/conversation-view/history-writer'; +import { materializeReplay } from '../src/materialize'; +import { buildSyntheticReplay, DEFAULT_SYNTHETIC_REPLAY } from './fixture'; + const BENCH_SESSION_ID = 'bench-session' as SessionId; +const PROVIDER = { cliType: 'builtin', agentType: 'claude' } as const; +const SCROLL_WINDOW = 30; +const SCROLL_ITERATIONS = 20; +const STREAM_ITERATIONS = 100; +const OPEN_BUDGET_MS = 50; +const STREAM_P99_BUDGET_MS = 4; function expandHome(value: string): string { return value.startsWith('~/') ? path.join(os.homedir(), value.slice(2)) : value; @@ -41,18 +75,32 @@ function readFlag(name: string): string | null { return hit ? hit.slice(prefix.length) : null; } -function loadFixture(): SessionHistoryInput[] { - const fixture = readFlag('fixture') ?? path.join(os.tmpdir(), 'lody-conversation-fixture.json'); - const file = expandHome(fixture); - const parsed = JSON.parse(readFileSync(file, 'utf8')) as SessionHistoryInput[]; - if (!Array.isArray(parsed) || parsed.length === 0) { - throw new Error(`${file} is not a non-empty history array; run bench:capture first`); +function loadFixture(): { history: SessionHistoryInput[]; source: string } { + const fixture = readFlag('fixture'); + if (fixture) { + const file = expandHome(fixture); + const parsed = JSON.parse(readFileSync(file, 'utf8')) as SessionHistoryInput[]; + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error(`${file} is not a non-empty history array; run bench:capture first`); + } + return { history: parsed, source: `captured (${path.basename(file)})` }; } - return parsed; + const turns = Number.parseInt(readFlag('turns') ?? String(DEFAULT_SYNTHETIC_REPLAY.turns), 10); + const materialized = materializeReplay({ + provider: PROVIDER, + acpSessionId: 'bench-acp-session' as never, + replayNotifications: buildSyntheticReplay({ ...DEFAULT_SYNTHETIC_REPLAY, turns }), + userId: 'bench-user', + nowIso: '2026-01-01T00:00:00.000Z', + }); + return { history: materialized.history, source: `synthetic (turns=${turns})` }; } /** `history` repeated `scale` times with unique entry ids. */ -function scaleHistory(history: readonly SessionHistoryInput[], scale: number): SessionHistoryInput[] { +function scaleHistory( + history: readonly SessionHistoryInput[], + scale: number +): SessionHistoryInput[] { if (scale === 1) return history as SessionHistoryInput[]; const out: SessionHistoryInput[] = []; for (let copy = 0; copy < scale; copy += 1) { @@ -110,8 +158,83 @@ function countContainers(history: readonly SessionHistoryInput[]): number { return containers; } +type RenderRow = + | { kind: 'message'; key: string; itemCount: number } + | { kind: 'placeholder'; key: string }; + +/** One row per turn, from the hydrated tail or the index — what the stream builds. */ +function renderRows(view: ConversationView): RenderRow[] { + const rows: RenderRow[] = []; + for (let i = 0; i < view.turnCount; i += 1) { + const turn = view.turn(i); + if (turn) { + rows.push({ + kind: 'message', + key: turn.id, + itemCount: Array.isArray(turn.items) ? turn.items.length : 0, + }); + continue; + } + const row = view.index(i); + if (row?.id) rows.push({ kind: 'placeholder', key: row.id }); + } + return rows; +} + +/** The last `LoroText` item of the last turn: where a streamed token lands. */ +function tailTextContainer(doc: LoroDoc): LoroText { + const cids = doc.getList('history').getShallowValue() as string[]; + for (let turnIndex = cids.length - 1; turnIndex >= 0; turnIndex -= 1) { + const turn = doc.getContainerById(cids[turnIndex] as never); + if (!(turn instanceof LoroMap)) continue; + const items = turn.get('items'); + if (!(items instanceof LoroList)) continue; + for (let i = items.length - 1; i >= 0; i -= 1) { + const item = items.get(i); + if (!(item instanceof LoroMap)) continue; + const text = item.get('text'); + if (text instanceof LoroText) return text; + } + } + throw new Error('fixture has no text item to stream into'); +} + +const appendedUserTurn = (index: number): SessionHistoryInput => ({ + id: `bench-append-${index}`, + role: 'user', + timestamp: '2026-01-01T00:00:00.000Z', + status: 'pending', + read: false, + finished: true, + fileDiff: [], + items: [{ type: 'text', text: `follow-up ${index}` }] as never, + inputConfig: { prompt: `follow-up ${index}`, cliType: 'builtin', agentType: 'claude' } as never, +}); + +type Row = { + task: string; + 'mean ms': number; + 'p99 ms': number; + 'ops/sec': number; + samples: number; +}; + +const rowsOf = (bench: Bench): Row[] => + bench.tasks.map((task) => ({ + task: task.name, + 'mean ms': Number((task.result?.mean ?? 0).toFixed(2)), + 'p99 ms': Number((task.result?.p99 ?? 0).toFixed(2)), + 'ops/sec': Number((task.result?.hz ?? 0).toFixed(1)), + samples: task.result?.samples.length ?? 0, + })); + +const meanOf = (bench: Bench, name: string): number => + bench.tasks.find((task) => task.name === name)?.result?.mean ?? 0; +const p99Of = (bench: Bench, name: string): number => + bench.tasks.find((task) => task.name === name)?.result?.p99 ?? 0; + async function main(): Promise { - const base = loadFixture(); + const { history: base, source } = loadFixture(); const scales = (readFlag('scale') ?? '1,10') .split(',') .map((value) => Number.parseInt(value.trim(), 10)) @@ -119,12 +242,15 @@ async function main(): Promise { const iterations = Number.parseInt(readFlag('iterations') ?? '10', 10); process.stdout.write( - `fixture: ${base.length} turns, ${countItems(base)} message items\n` + + `fixture: ${source}, ${base.length} turns, ${countItems(base)} message items\n` + `scales: ${scales.join(', ')} iterations: ${iterations}\n\n` ); + const progress = (message: string) => process.stderr.write(` … ${message}\n`); + for (const scale of scales) { const history = scaleHistory(base, scale); + progress(`x${scale}: building snapshot (${history.length} turns)`); const snapshot = buildSnapshot(history); const containers = countContainers(history); const preImported = importedDoc(snapshot); @@ -133,9 +259,103 @@ async function main(): Promise { // `time: 0` with an explicit iteration count keeps every task at exactly the // requested number of runs: a 100x doc takes seconds per run, and tinybench's // default time budget would otherwise decide the sample size for us. - const bench = new Bench({ time: 0, iterations, warmupIterations: 1 }); + // View tasks run first: the Mirror baseline allocates the whole + // transcript per iteration, and measuring the view on that heap would + // charge its garbage collection to the view. + const viewBench = new Bench({ time: 0, iterations, warmupIterations: 1 }); + viewBench + .add('view.open', () => { + const view = createConversationViewFromDoc(importedDoc(snapshot), { + sessionId: BENCH_SESSION_ID, + }); + renderRows(view); + view.dispose(); + }) + .add('view.readAll', () => { + const view = createConversationViewFromDoc(importedDoc(snapshot), { + sessionId: BENCH_SESSION_ID, + }); + view.readAll(); + view.dispose(); + }); + progress(`x${scale}: view.open / view.readAll`); + await viewBench.run(); + + let scrollView: ConversationView | null = null; + let cursor = 0; + const scrollBench = new Bench({ time: 0, iterations: SCROLL_ITERATIONS, warmupIterations: 0 }); + scrollBench.add( + 'view.scroll', + async () => { + const view = scrollView; + if (!view) return; + await view.ensureRange(cursor, cursor + SCROLL_WINDOW); + cursor = (cursor + SCROLL_WINDOW) % Math.max(1, view.turnCount - SCROLL_WINDOW); + }, + { + beforeAll: () => { + scrollView = createConversationViewFromDoc(preImported, { sessionId: BENCH_SESSION_ID }); + cursor = 0; + }, + afterAll: () => { + scrollView?.dispose(); + scrollView = null; + }, + } + ); + progress(`x${scale}: scroll`); + await scrollBench.run(); - bench + let streamDoc: LoroDoc | null = null; + let streamView: ConversationView | null = null; + let streamText: LoroText | null = null; + let appended = 0; + const streamBench = new Bench({ time: 0, iterations: STREAM_ITERATIONS, warmupIterations: 0 }); + streamBench + .add( + 'view.stream', + () => { + if (!streamDoc || !streamText) return; + streamText.insert(streamText.length, ' token'); + streamDoc.commit(); + }, + { + beforeAll: () => { + streamDoc = importedDoc(snapshot); + streamView = createConversationViewFromDoc(streamDoc, { sessionId: BENCH_SESSION_ID }); + streamText = tailTextContainer(streamDoc); + }, + afterAll: () => { + streamView?.dispose(); + streamView = null; + streamText = null; + }, + } + ) + .add( + 'view.append', + () => { + if (!streamDoc) return; + appendHistoryEntry(streamDoc, appendedUserTurn((appended += 1))); + }, + { + beforeAll: () => { + streamDoc = importedDoc(snapshot); + streamView = createConversationViewFromDoc(streamDoc, { sessionId: BENCH_SESSION_ID }); + appended = 0; + }, + afterAll: () => { + streamView?.dispose(); + streamView = null; + streamDoc = null; + }, + } + ); + progress(`x${scale}: stream + append`); + await streamBench.run(); + + const openBench = new Bench({ time: 0, iterations, warmupIterations: 1 }); + openBench .add('import', () => { importedDoc(snapshot); }) @@ -156,26 +376,29 @@ async function main(): Promise { .add('getState', () => { mirror?.getState(); }); - - await bench.run(); - - const rows = bench.tasks.map((task) => ({ - task: task.name, - 'mean ms': Number((task.result?.mean ?? 0).toFixed(1)), - 'p99 ms': Number((task.result?.p99 ?? 0).toFixed(1)), - 'ops/sec': Number((task.result?.hz ?? 0).toFixed(1)), - samples: task.result?.samples.length ?? 0, - })); + progress(`x${scale}: baseline (import / toJSON / Mirror)`); + await openBench.run(); process.stdout.write( `scale x${scale}: ${history.length} turns, ${countItems(history)} items, ` + `${containers} containers, snapshot ${(snapshot.byteLength / 1024 / 1024).toFixed(1)} MiB\n` ); - console.table(rows); - const mirrorMean = bench.tasks.find((task) => task.name === 'Mirror')?.result?.mean ?? 0; + console.table([ + ...rowsOf(openBench), + ...rowsOf(viewBench), + ...rowsOf(scrollBench), + ...rowsOf(streamBench), + ]); + const mirrorMean = meanOf(openBench, 'Mirror'); + const openMean = meanOf(viewBench, 'view.open'); + const streamP99 = p99Of(streamBench, 'view.stream'); process.stdout.write( - ` ${(mirrorMean * 1000).toFixed(0)} µs/turn, ` + - `${((mirrorMean * 1000) / Math.max(containers, 1)).toFixed(1)} µs/container\n\n` + ` before (Mirror): ${mirrorMean.toFixed(1)} ms = ` + + `${((mirrorMean * 1000) / Math.max(history.length, 1)).toFixed(0)} µs/turn, ` + + `${((mirrorMean * 1000) / Math.max(containers, 1)).toFixed(1)} µs/container\n` + + ` after (view.open): ${openMean.toFixed(1)} ms (${(mirrorMean / Math.max(openMean, 1e-6)).toFixed(1)}x faster)` + + ` open<=${OPEN_BUDGET_MS}ms: ${openMean <= OPEN_BUDGET_MS ? 'PASS' : 'FAIL'}` + + ` stream p99<=${STREAM_P99_BUDGET_MS}ms: ${streamP99 <= STREAM_P99_BUDGET_MS ? 'PASS' : 'FAIL'} (${streamP99.toFixed(2)} ms)\n\n` ); } } From be3c1215976d0f36b350b7f631b9702982da2724 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Fri, 4 Sep 2026 07:59:43 +0800 Subject: [PATCH 2/2] chore(history-import): report an errored Mirror baseline instead of crashing the bench summary The full-schema Mirror throws `unreachable` on the desensitized real fixture at x10 (570 turns, 56k items); the reporter now prints the error per task and in the summary. Model: claude-fable-5-1 --- .../benchmarks/open-conversation.bench.ts | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/history-import/benchmarks/open-conversation.bench.ts b/packages/history-import/benchmarks/open-conversation.bench.ts index 55c9bf7e5..622647231 100644 --- a/packages/history-import/benchmarks/open-conversation.bench.ts +++ b/packages/history-import/benchmarks/open-conversation.bench.ts @@ -217,6 +217,7 @@ type Row = { 'p99 ms': number; 'ops/sec': number; samples: number; + error?: string; }; const rowsOf = (bench: Bench): Row[] => @@ -225,7 +226,12 @@ const rowsOf = (bench: Bench): Row[] => 'mean ms': Number((task.result?.mean ?? 0).toFixed(2)), 'p99 ms': Number((task.result?.p99 ?? 0).toFixed(2)), 'ops/sec': Number((task.result?.hz ?? 0).toFixed(1)), - samples: task.result?.samples.length ?? 0, + samples: task.result?.samples?.length ?? 0, + // A task that threw (the full Mirror on a large doc can) has `result.error` + // and no samples; report it instead of crashing the whole run. + ...(task.result?.error + ? { error: String((task.result.error as { message?: string }).message ?? task.result.error) } + : {}), })); const meanOf = (bench: Bench, name: string): number => @@ -390,13 +396,20 @@ async function main(): Promise { ...rowsOf(streamBench), ]); const mirrorMean = meanOf(openBench, 'Mirror'); + const mirrorError = openBench.tasks.find((task) => task.name === 'Mirror')?.result?.error; const openMean = meanOf(viewBench, 'view.open'); const streamP99 = p99Of(streamBench, 'view.stream'); - process.stdout.write( - ` before (Mirror): ${mirrorMean.toFixed(1)} ms = ` + + const before = mirrorError + ? ` before (Mirror): FAILED (${String((mirrorError as { message?: string }).message ?? mirrorError)})\n` + : ` before (Mirror): ${mirrorMean.toFixed(1)} ms = ` + `${((mirrorMean * 1000) / Math.max(history.length, 1)).toFixed(0)} µs/turn, ` + - `${((mirrorMean * 1000) / Math.max(containers, 1)).toFixed(1)} µs/container\n` + - ` after (view.open): ${openMean.toFixed(1)} ms (${(mirrorMean / Math.max(openMean, 1e-6)).toFixed(1)}x faster)` + + `${((mirrorMean * 1000) / Math.max(containers, 1)).toFixed(1)} µs/container\n`; + const speedup = mirrorError + ? '' + : ` (${(mirrorMean / Math.max(openMean, 1e-6)).toFixed(1)}x faster)`; + process.stdout.write( + before + + ` after (view.open): ${openMean.toFixed(1)} ms${speedup}` + ` open<=${OPEN_BUDGET_MS}ms: ${openMean <= OPEN_BUDGET_MS ? 'PASS' : 'FAIL'}` + ` stream p99<=${STREAM_P99_BUDGET_MS}ms: ${streamP99 <= STREAM_P99_BUDGET_MS ? 'PASS' : 'FAIL'} (${streamP99.toFixed(2)} ms)\n\n` );