From bd7e506596018e4a550775402f7b3e094a15b816 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:44:30 -0700 Subject: [PATCH 001/260] test(perf): require deterministic synthetic benchmark corpus --- src/performanceCorpusContract.test.ts | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/performanceCorpusContract.test.ts diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts new file mode 100644 index 00000000..5c2d6edd --- /dev/null +++ b/src/performanceCorpusContract.test.ts @@ -0,0 +1,77 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkProfileLock { + readonly sections: number; + readonly bytes: number; + readonly sha256: string; +} + +interface BenchmarkCorpusLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly scripts: readonly [ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]; + readonly profiles: Readonly< + Record<'small' | 'medium' | 'large' | 'stress', BenchmarkProfileLock> + >; +} + +function runGenerator(outputDirectory: string): BenchmarkCorpusLock { + const script = resolve(process.cwd(), 'benchmarks/generate-corpus.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as BenchmarkCorpusLock; +} + +describe('deterministic synthetic performance corpus', () => { + it('reproduces the committed corpus lock exactly across independent runs', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/corpus.lock.json'), + 'utf8', + ), + ) as BenchmarkCorpusLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest.synthetic).toBe(true); + expect(firstManifest.scripts).toEqual([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]); + + for (const profile of ['small', 'medium', 'large', 'stress'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.md`)); + const secondBytes = readFileSync(join(second, `${profile}.md`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From e913b9e0da7d9314aa989b2ba5a5d1c26badb85b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:44:54 -0700 Subject: [PATCH 002/260] feat(perf): add deterministic multilingual corpus generator --- benchmarks/generate-corpus.mjs | 112 +++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 benchmarks/generate-corpus.mjs diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs new file mode 100644 index 00000000..4f582805 --- /dev/null +++ b/benchmarks/generate-corpus.mjs @@ -0,0 +1,112 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const PIXEL_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; +const SCRIPT_PARAGRAPHS = [ + 'English: Deterministic authoring keeps document state explicit and reviewable.', + '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', + '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', + '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', + 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', +]; +const PROFILE_SECTIONS = Object.freeze({ + small: 1, + medium: 8, + large: 32, + stress: 128, +}); +const SCRIPT_LABELS = Object.freeze([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', +]); + +function buildSection(index) { + const id = String(index).padStart(4, '0'); + const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { + const row = String(rowIndex + 1).padStart(2, '0'); + return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; + }); + return [ + `# Synthetic section ${id}`, + '', + ...SCRIPT_PARAGRAPHS, + '', + `## Nested list ${id}`, + `- item ${id}-a`, + ` - item ${id}-a-1`, + ` - item ${id}-a-2`, + `- item ${id}-b`, + '', + `> Synthetic blockquote ${id}: benchmark text only; no production content.`, + '', + '```text', + `fixture=${id}; authority=none; network=none`, + '```', + '', + `[synthetic safe link ${id}](https://example.invalid/inkspan/${id})`, + '', + '| c01 | c02 | c03 | c04 | c05 | c06 |', + '| --- | --- | --- | --- | --- | --- |', + ...tableRows, + '', + `![synthetic 1x1 raster ${id}](data:image/png;base64,${PIXEL_BASE64})`, + '', + '---', + '', + ].join('\n'); +} + +function buildProfile(profile, sectionCount) { + return [ + `# Inkspan deterministic benchmark fixture: ${profile}`, + '', + 'Synthetic fixture only. No customer, tenant, prompt, credential, or model data.', + 'Scripts: English, Korean, Japanese, Chinese, Vietnamese, and mixed-script structure.', + '', + ...Array.from({ length: sectionCount }, (_, index) => buildSection(index + 1)), + ].join('\n'); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error('Usage: node benchmarks/generate-corpus.mjs --output '); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const profileManifest = {}; +for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { + const body = buildProfile(profile, sections); + const bytes = Buffer.from(body, 'utf8'); + writeFileSync(resolve(outputDirectory, `${profile}.md`), bytes); + profileManifest[profile] = Object.freeze({ + sections, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + scripts: SCRIPT_LABELS, + profiles: profileManifest, +}); +writeFileSync( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', +); From 0f941418de9bfa90c5ff1edf007017dc814b2605 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:45:08 -0700 Subject: [PATCH 003/260] test(perf): lock synthetic corpus identities --- benchmarks/corpus.lock.json | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 benchmarks/corpus.lock.json diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json new file mode 100644 index 00000000..15a68e1c --- /dev/null +++ b/benchmarks/corpus.lock.json @@ -0,0 +1,34 @@ +{ + "contractVersion": 1, + "synthetic": true, + "scripts": [ + "English", + "Korean", + "Japanese", + "Chinese", + "Vietnamese", + "mixed" + ], + "profiles": { + "small": { + "sections": 1, + "bytes": 1577, + "sha256": "768c220ae809f29fd9e20234f55de93f05a532fe3a6844a083b8d9d1d80af2e7" + }, + "medium": { + "sections": 8, + "bytes": 11112, + "sha256": "5d1cebaf4e87374d2627a629dc7afc80cbc67e94f7ca447702eb3f836c6f4f50" + }, + "large": { + "sections": 32, + "bytes": 43799, + "sha256": "24bd29e0f4860d7a3ce44aaef74ddf4e0c6e747254b683000136500fd93487ed" + }, + "stress": { + "sections": 128, + "bytes": 174552, + "sha256": "ae486d9257ca3c227233833feef1d4a2a7f1e0110dd4e1973d5748f7873c53db" + } + } +} From b94f1246d6efebf345ae6cbb8efdcc558e4fc579 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:17:15 -0700 Subject: [PATCH 004/260] test(perf): require mixed script and raster size corpus --- src/performanceCorpusContract.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 5c2d6edd..7ab91cca 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -70,6 +70,17 @@ describe('deterministic synthetic performance corpus', () => { expect(firstBytes.equals(secondBytes)).toBe(true); expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); } + + const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); + expect(smallBody).toContain( + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', + ); + for (const dimensions of ['1x1', '16x16', '64x64']) { + expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); + } + expect(new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g))).toHaveLength( + 3, + ); } finally { rmSync(root, { recursive: true, force: true }); } From 1ff70213a49fb15778d0ae9d74a5dcdef101f87e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:18:19 -0700 Subject: [PATCH 005/260] feat(perf): add mixed script and raster size fixtures --- benchmarks/generate-corpus.mjs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index 4f582805..af73290e 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -2,14 +2,30 @@ import { createHash } from 'node:crypto'; import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; -const PIXEL_BASE64 = - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; +const RASTER_FIXTURES = Object.freeze([ + Object.freeze({ + dimensions: '1x1', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNQcEj4DwADBAHAX3ZiygAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '16x16', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR42mNQcEj4TwlmGDVg1IBRA4aLAQDSpr8QG8NsyQAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '64x64', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAZklEQVR42u3QQREAAAQAMFFEEUX/EuRw9liBRVbPZyFAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgTct/Bk8ZbOy3oMAAAAAElFTkSuQmCC', + }), +]); const SCRIPT_PARAGRAPHS = [ 'English: Deterministic authoring keeps document state explicit and reviewable.', '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', ]; const PROFILE_SECTIONS = Object.freeze({ small: 1, @@ -32,6 +48,10 @@ function buildSection(index) { const row = String(rowIndex + 1).padStart(2, '0'); return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; }); + const rasterRows = RASTER_FIXTURES.map( + ({ dimensions, base64 }) => + `![synthetic raster ${dimensions} ${id}](data:image/png;base64,${base64})`, + ); return [ `# Synthetic section ${id}`, '', @@ -55,7 +75,7 @@ function buildSection(index) { '| --- | --- | --- | --- | --- | --- |', ...tableRows, '', - `![synthetic 1x1 raster ${id}](data:image/png;base64,${PIXEL_BASE64})`, + ...rasterRows, '', '---', '', From 322dc641173073244e70f861601e5c7df75ffa22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:18:36 -0700 Subject: [PATCH 006/260] test(perf): lock mixed script corpus identities --- benchmarks/corpus.lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index 15a68e1c..fa2fe87c 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -12,23 +12,23 @@ "profiles": { "small": { "sections": 1, - "bytes": 1577, - "sha256": "768c220ae809f29fd9e20234f55de93f05a532fe3a6844a083b8d9d1d80af2e7" + "bytes": 2152, + "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea" }, "medium": { "sections": 8, - "bytes": 11112, - "sha256": "5d1cebaf4e87374d2627a629dc7afc80cbc67e94f7ca447702eb3f836c6f4f50" + "bytes": 15712, + "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c" }, "large": { "sections": 32, - "bytes": 43799, - "sha256": "24bd29e0f4860d7a3ce44aaef74ddf4e0c6e747254b683000136500fd93487ed" + "bytes": 62199, + "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392" }, "stress": { "sections": 128, - "bytes": 174552, - "sha256": "ae486d9257ca3c227233833feef1d4a2a7f1e0110dd4e1973d5748f7873c53db" + "bytes": 248152, + "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928" } } } From 45a74f8f9cb01033d73a111b6a3f602acd96fb37 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:20:26 -0700 Subject: [PATCH 007/260] test(perf): assert distinct raster fixture count --- src/performanceCorpusContract.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 7ab91cca..e92c7f2c 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -78,9 +78,9 @@ describe('deterministic synthetic performance corpus', () => { for (const dimensions of ['1x1', '16x16', '64x64']) { expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); } - expect(new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g))).toHaveLength( - 3, - ); + expect( + new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g)).size, + ).toBe(3); } finally { rmSync(root, { recursive: true, force: true }); } From 5831e7c76986ee2984468bb7926caa709faa31eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:01:18 -0700 Subject: [PATCH 008/260] test(perf): require deterministic Office benchmark fixtures --- src/performanceOfficeFixtureContract.test.ts | 82 ++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/performanceOfficeFixtureContract.test.ts diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts new file mode 100644 index 00000000..d8a9094d --- /dev/null +++ b/src/performanceOfficeFixtureContract.test.ts @@ -0,0 +1,82 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface OfficeFixtureProfileLock { + readonly pages: number; + readonly blocks: number; + readonly bytes: number; + readonly sha256: string; +} + +interface OfficeFixtureLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly format: 'docx'; + readonly profiles: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; +} + +function runGenerator(outputDirectory: string): OfficeFixtureLock { + const script = resolve(process.cwd(), 'benchmarks/generate-office-fixtures.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as OfficeFixtureLock; +} + +describe('deterministic synthetic Office performance fixtures', () => { + it('reproduces a schema-shaped DOCX corpus including a 120-page fixture', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/office-fixtures.lock.json'), + 'utf8', + ), + ) as OfficeFixtureLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest).toEqual({ + contractVersion: 1, + synthetic: true, + format: 'docx', + profiles: expected.profiles, + }); + + for (const profile of ['small', 'page120'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.json`)); + const secondBytes = readFileSync(join(second, `${profile}.json`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + } + + const page120 = JSON.parse( + readFileSync(join(first, 'page120.json'), 'utf8'), + ) as { + format: string; + blocks: Array<{ type: string; text?: string }>; + }; + expect(page120.format).toBe('docx'); + expect(page120.blocks.filter(({ type }) => type === 'heading')).toHaveLength(120); + expect(page120.blocks.filter(({ type }) => type === 'page_break')).toHaveLength(119); + expect(page120.blocks.some(({ text }) => text?.includes('한국어'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 4f702a3d60152a904d6c64f480a34f38e9787d8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:03:35 -0700 Subject: [PATCH 009/260] perf: generate deterministic Office benchmark fixtures --- benchmarks/generate-office-fixtures.mjs | 113 ++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 benchmarks/generate-office-fixtures.mjs diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs new file mode 100644 index 00000000..76d71a4a --- /dev/null +++ b/benchmarks/generate-office-fixtures.mjs @@ -0,0 +1,113 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const PROFILE_PAGES = Object.freeze({ + small: 2, + page120: 120, +}); + +const MULTILINGUAL_PARAGRAPH = + 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; + +function buildPage(pageNumber) { + const page = String(pageNumber).padStart(3, '0'); + return [ + Object.freeze({ + type: 'heading', + level: 1, + text: `Synthetic page ${page}`, + }), + Object.freeze({ + type: 'paragraph', + text: `${MULTILINGUAL_PARAGRAPH} Page ${page}.`, + alignment: 'justify', + }), + Object.freeze({ + type: 'rich_paragraph', + runs: Object.freeze([ + Object.freeze({ text: `Page ${page} summary: `, bold: true }), + Object.freeze({ text: 'deterministic ', italic: true }), + Object.freeze({ text: 'Office rendering fixture.', underline: true }), + ]), + }), + Object.freeze({ + type: 'bullet_list', + ordered: false, + items: Object.freeze([ + `page ${page} item A`, + `page ${page} item B`, + `page ${page} item C`, + ]), + }), + Object.freeze({ + type: 'table', + headers: Object.freeze(['Page', 'Metric', 'Value']), + rows: Object.freeze([ + Object.freeze([page, 'latency-sample', pageNumber]), + Object.freeze([page, 'memory-sample', pageNumber * 2]), + Object.freeze([page, 'revision-sample', pageNumber * 3]), + Object.freeze([page, 'render-sample', pageNumber * 4]), + ]), + }), + ]; +} + +function buildRequest(profile, pages) { + const blocks = []; + for (let page = 1; page <= pages; page += 1) { + blocks.push(...buildPage(page)); + if (page < pages) { + blocks.push(Object.freeze({ type: 'page_break' })); + } + } + return Object.freeze({ + format: 'docx', + title: `Inkspan synthetic Office benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + subject: 'Deterministic synthetic performance fixture', + blocks: Object.freeze(blocks), + }); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error( + 'Usage: node benchmarks/generate-office-fixtures.mjs --output ', + ); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const profileManifest = {}; +for (const [profile, pages] of Object.entries(PROFILE_PAGES)) { + const request = buildRequest(profile, pages); + const body = `${JSON.stringify(request, null, 2)}\n`; + const bytes = Buffer.from(body, 'utf8'); + writeFileSync(resolve(outputDirectory, `${profile}.json`), bytes); + profileManifest[profile] = Object.freeze({ + pages, + blocks: request.blocks.length, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + format: 'docx', + profiles: profileManifest, +}); +writeFileSync( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', +); From bf1fe4e8b17ac346bc827e47bf036c80c306e1ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:04:01 -0700 Subject: [PATCH 010/260] perf: lock deterministic Office benchmark fixtures --- benchmarks/office-fixtures.lock.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 benchmarks/office-fixtures.lock.json diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json new file mode 100644 index 00000000..f3ff4beb --- /dev/null +++ b/benchmarks/office-fixtures.lock.json @@ -0,0 +1,19 @@ +{ + "contractVersion": 1, + "synthetic": true, + "format": "docx", + "profiles": { + "small": { + "pages": 2, + "blocks": 11, + "bytes": 2974, + "sha256": "9255bd3136bd5523169e495c365235a8b7bb7152092145ae41e2867f92dcc71a" + }, + "page120": { + "pages": 120, + "blocks": 719, + "bytes": 169739, + "sha256": "4df660c5ad762a516457d3005537861acf7c17478269744fca747e84fcbed3f9" + } + } +} From 0a67a3eec7035449a4973288d2b12f6e4e35dfa3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:11:16 -0700 Subject: [PATCH 011/260] test(perf): require XLSX and PPTX benchmark fixtures --- src/performanceOfficeFixtureContract.test.ts | 93 +++++++++++++++++--- 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index d8a9094d..ebc1fa2d 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -5,8 +5,7 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; interface OfficeFixtureProfileLock { - readonly pages: number; - readonly blocks: number; + readonly units: number; readonly bytes: number; readonly sha256: string; } @@ -14,10 +13,17 @@ interface OfficeFixtureProfileLock { interface OfficeFixtureLock { readonly contractVersion: 1; readonly synthetic: true; - readonly format: 'docx'; - readonly profiles: Readonly< - Record<'small' | 'page120', OfficeFixtureProfileLock> - >; + readonly formats: Readonly<{ + docx: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; + xlsx: Readonly< + Record<'small' | 'wide16384', OfficeFixtureProfileLock> + >; + pptx: Readonly< + Record<'small' | 'slide120', OfficeFixtureProfileLock> + >; + }>; } function runGenerator(outputDirectory: string): OfficeFixtureLock { @@ -31,8 +37,20 @@ function runGenerator(outputDirectory: string): OfficeFixtureLock { ) as OfficeFixtureLock; } +function expectDeterministicFixture( + first: string, + second: string, + fileName: string, + expectedBytes: number, +): void { + const firstBytes = readFileSync(join(first, fileName)); + const secondBytes = readFileSync(join(second, fileName)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expectedBytes); +} + describe('deterministic synthetic Office performance fixtures', () => { - it('reproduces a schema-shaped DOCX corpus including a 120-page fixture', () => { + it('reproduces bounded DOCX, XLSX, and PPTX corpora including 100+ unit fixtures', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); const first = join(root, 'first'); const second = join(root, 'second'); @@ -51,19 +69,36 @@ describe('deterministic synthetic Office performance fixtures', () => { expect(firstManifest).toEqual({ contractVersion: 1, synthetic: true, - format: 'docx', - profiles: expected.profiles, + formats: expected.formats, }); for (const profile of ['small', 'page120'] as const) { - const firstBytes = readFileSync(join(first, `${profile}.json`)); - const secondBytes = readFileSync(join(second, `${profile}.json`)); - expect(firstBytes.equals(secondBytes)).toBe(true); - expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + expectDeterministicFixture( + first, + second, + `docx-${profile}.json`, + expected.formats.docx[profile].bytes, + ); + } + for (const profile of ['small', 'wide16384'] as const) { + expectDeterministicFixture( + first, + second, + `xlsx-${profile}.json`, + expected.formats.xlsx[profile].bytes, + ); + } + for (const profile of ['small', 'slide120'] as const) { + expectDeterministicFixture( + first, + second, + `pptx-${profile}.json`, + expected.formats.pptx[profile].bytes, + ); } const page120 = JSON.parse( - readFileSync(join(first, 'page120.json'), 'utf8'), + readFileSync(join(first, 'docx-page120.json'), 'utf8'), ) as { format: string; blocks: Array<{ type: string; text?: string }>; @@ -75,6 +110,36 @@ describe('deterministic synthetic Office performance fixtures', () => { expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + + const wide = JSON.parse( + readFileSync(join(first, 'xlsx-wide16384.json'), 'utf8'), + ) as { + format: string; + sheets: Array<{ rows: unknown[][]; freeze_panes?: string }>; + }; + expect(wide.format).toBe('xlsx'); + expect(wide.sheets).toHaveLength(1); + expect(wide.sheets[0]?.rows[0]).toHaveLength(16_384); + expect(wide.sheets[0]?.freeze_panes).toBe('XFD1048576'); + + const slide120 = JSON.parse( + readFileSync(join(first, 'pptx-slide120.json'), 'utf8'), + ) as { + format: string; + slides: Array<{ title: string; bullets?: Array }>; + }; + expect(slide120.format).toBe('pptx'); + expect(slide120.slides).toHaveLength(120); + expect(slide120.slides.some(({ title }) => title.includes('한국어'))).toBe(true); + expect( + slide120.slides.some(({ bullets }) => + bullets?.some((bullet) => + typeof bullet === 'string' + ? bullet.includes('日本語') + : bullet.text.includes('日本語'), + ), + ), + ).toBe(true); } finally { rmSync(root, { recursive: true, force: true }); } From f6e795d650433066cc319d220b796c76b987ab94 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:12:02 -0700 Subject: [PATCH 012/260] perf: generate deterministic XLSX and PPTX fixtures --- benchmarks/generate-office-fixtures.mjs | 140 +++++++++++++++++++++--- 1 file changed, 123 insertions(+), 17 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 76d71a4a..40a512df 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -2,15 +2,21 @@ import { createHash } from 'node:crypto'; import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; -const PROFILE_PAGES = Object.freeze({ +const DOCX_PROFILE_PAGES = Object.freeze({ small: 2, page120: 120, }); +const PPTX_PROFILE_SLIDES = Object.freeze({ + small: 2, + slide120: 120, +}); + +const EXCEL_MAX_COLUMNS = 16_384; const MULTILINGUAL_PARAGRAPH = 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; -function buildPage(pageNumber) { +function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); return [ Object.freeze({ @@ -53,23 +59,88 @@ function buildPage(pageNumber) { ]; } -function buildRequest(profile, pages) { +function buildDocxRequest(profile, pages) { const blocks = []; for (let page = 1; page <= pages; page += 1) { - blocks.push(...buildPage(page)); + blocks.push(...buildDocxPage(page)); if (page < pages) { blocks.push(Object.freeze({ type: 'page_break' })); } } return Object.freeze({ format: 'docx', - title: `Inkspan synthetic Office benchmark: ${profile}`, + title: `Inkspan synthetic DOCX benchmark: ${profile}`, author: 'Inkspan synthetic benchmark', subject: 'Deterministic synthetic performance fixture', blocks: Object.freeze(blocks), }); } +function buildXlsxRequest(profile) { + if (profile === 'wide16384') { + const row = Array.from( + { length: EXCEL_MAX_COLUMNS }, + (_, index) => `C${String(index + 1).padStart(5, '0')}`, + ); + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: wide16384', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Wide16384', + rows: Object.freeze([Object.freeze(row)]), + freeze_panes: 'XFD1048576', + }), + ]), + }); + } + + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: small', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Synthetic', + header_row: true, + auto_filter: true, + freeze_panes: 'B2', + rows: Object.freeze([ + Object.freeze(['Language', 'Text', 'Latency', 'Memory']), + Object.freeze(['한국어', '합성 성능 문서', 1, 2]), + Object.freeze(['日本語', '合成性能文書', 3, 4]), + Object.freeze(['中文 / Tiếng Việt', '合成文档 / tài liệu tổng hợp', 5, 6]), + ]), + }), + ]), + }); +} + +function buildPptxSlide(slideNumber) { + const slide = String(slideNumber).padStart(3, '0'); + return Object.freeze({ + title: `한국어 합성 슬라이드 ${slide}`, + bullets: Object.freeze([ + `English deterministic slide ${slide}`, + Object.freeze({ text: `日本語 合成スライド ${slide}`, level: 0 }), + Object.freeze({ text: `中文 合成幻灯片 ${slide}`, level: 1 }), + Object.freeze({ text: `Tiếng Việt trang chiếu ${slide}`, level: 1 }), + ]), + }); +} + +function buildPptxRequest(profile, slides) { + return Object.freeze({ + format: 'pptx', + title: `Inkspan synthetic PPTX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + slides: Object.freeze( + Array.from({ length: slides }, (_, index) => buildPptxSlide(index + 1)), + ), + }); +} + function sha256(bytes) { return createHash('sha256').update(bytes).digest('hex'); } @@ -83,28 +154,63 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } -const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); -mkdirSync(outputDirectory, { recursive: true }); - -const profileManifest = {}; -for (const [profile, pages] of Object.entries(PROFILE_PAGES)) { - const request = buildRequest(profile, pages); +function writeFixture(outputDirectory, fileName, request, units) { const body = `${JSON.stringify(request, null, 2)}\n`; const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, `${profile}.json`), bytes); - profileManifest[profile] = Object.freeze({ - pages, - blocks: request.blocks.length, + writeFileSync(resolve(outputDirectory, fileName), bytes); + return Object.freeze({ + units, bytes: bytes.byteLength, sha256: sha256(bytes), }); } +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const docx = {}; +for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { + docx[profile] = writeFixture( + outputDirectory, + `docx-${profile}.json`, + buildDocxRequest(profile, pages), + pages, + ); +} + +const xlsx = Object.freeze({ + small: writeFixture( + outputDirectory, + 'xlsx-small.json', + buildXlsxRequest('small'), + 4, + ), + wide16384: writeFixture( + outputDirectory, + 'xlsx-wide16384.json', + buildXlsxRequest('wide16384'), + EXCEL_MAX_COLUMNS, + ), +}); + +const pptx = {}; +for (const [profile, slides] of Object.entries(PPTX_PROFILE_SLIDES)) { + pptx[profile] = writeFixture( + outputDirectory, + `pptx-${profile}.json`, + buildPptxRequest(profile, slides), + slides, + ); +} + const manifest = Object.freeze({ contractVersion: 1, synthetic: true, - format: 'docx', - profiles: profileManifest, + formats: Object.freeze({ + docx: Object.freeze(docx), + xlsx, + pptx: Object.freeze(pptx), + }), }); writeFileSync( resolve(outputDirectory, 'manifest.json'), From 940275af5f0ec3c191ac01dee78f691cc557fe1a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:12:45 -0700 Subject: [PATCH 013/260] perf: lock deterministic Office fixture matrix --- benchmarks/office-fixtures.lock.json | 47 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json index f3ff4beb..01f9920e 100644 --- a/benchmarks/office-fixtures.lock.json +++ b/benchmarks/office-fixtures.lock.json @@ -1,19 +1,42 @@ { "contractVersion": 1, "synthetic": true, - "format": "docx", - "profiles": { - "small": { - "pages": 2, - "blocks": 11, - "bytes": 2974, - "sha256": "9255bd3136bd5523169e495c365235a8b7bb7152092145ae41e2867f92dcc71a" + "formats": { + "docx": { + "small": { + "units": 2, + "bytes": 2972, + "sha256": "c356496b106f5348e98b00d3c1e18185165653a2c2bda083acd7456c96b2eab3" + }, + "page120": { + "units": 120, + "bytes": 169737, + "sha256": "e5d90408c6061051ab1674d931b99d478fadb00a7ecbcb7025b4fa478cfbb507" + } }, - "page120": { - "pages": 120, - "blocks": 719, - "bytes": 169739, - "sha256": "4df660c5ad762a516457d3005537861acf7c17478269744fca747e84fcbed3f9" + "xlsx": { + "small": { + "units": 4, + "bytes": 723, + "sha256": "a267a6208de0e24559c200d047404d14e2263581e0a3af845f355817536883bf" + }, + "wide16384": { + "units": 16384, + "bytes": 327941, + "sha256": "0afdb216fda720cd506d7c24284c2740f326167d4859dc1a7b845c4a91b972ec" + } + }, + "pptx": { + "small": { + "units": 2, + "bytes": 970, + "sha256": "6c1a8ae1d2307a278ac97903eb3160a6a0de57b695b2b8772844af2c8cfa2ce1" + }, + "slide120": { + "units": 120, + "bytes": 50061, + "sha256": "f5994ff30752581fd3b3e5210ff59b281692ea268ce4f0b540e278d134dbd6ee" + } } } } From 326e4ded064ecfc66ea0bf3d37f60fd746ddc383 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:06:20 -0700 Subject: [PATCH 014/260] test(perf): require deterministic benchmark statistics summary --- ...manceMeasurementStatisticsContract.test.ts | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/performanceMeasurementStatisticsContract.test.ts diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts new file mode 100644 index 00000000..6085aa25 --- /dev/null +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -0,0 +1,126 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSummary { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: string; + readonly sampleCount: number; + readonly percentileMethod: 'nearest-rank'; + readonly minimum: number; + readonly p50: number; + readonly p75: number; + readonly p95: number; + readonly maximum: number; +} + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string, samples: readonly number[]): void { + writeFileSync( + path, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummary { + execFileSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return JSON.parse( + readFileSync(join(outputDirectory, 'summary.json'), 'utf8'), + ) as BenchmarkSummary; +} + +describe('deterministic benchmark sample statistics', () => { + it('writes reproducible nearest-rank JSON and human-readable summaries', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); + const input = join(root, 'samples.json'); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + writeInput(input, [20, 10, 40, 30, 50]); + + const firstSummary = runSummary(input, first); + const secondSummary = runSummary(input, second); + expect(firstSummary).toEqual(secondSummary); + expect(firstSummary).toEqual({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sampleCount: 5, + percentileMethod: 'nearest-rank', + minimum: 10, + p50: 30, + p75: 40, + p95: 50, + maximum: 50, + }); + + const expectedText = [ + 'benchmark=markdown-serialization-large', + 'unit=ms', + 'samples=5', + 'percentile_method=nearest-rank', + 'minimum=10', + 'p50=30', + 'p75=40', + 'p95=50', + 'maximum=50', + '', + ].join('\n'); + expect(readFileSync(join(first, 'summary.txt'), 'utf8')).toBe(expectedText); + expect(readFileSync(join(second, 'summary.txt'), 'utf8')).toBe(expectedText); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on invalid measurement samples without coercion', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, -1, 3]); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark samples must be finite non-negative numbers.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 897a97838d544d41ab80f694889a6f22f3b9feb8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:06:55 -0700 Subject: [PATCH 015/260] feat(perf): add deterministic benchmark statistics summary --- benchmarks/summarize-samples.mjs | 139 +++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 benchmarks/summarize-samples.mjs diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs new file mode 100644 index 00000000..f695ae03 --- /dev/null +++ b/benchmarks/summarize-samples.mjs @@ -0,0 +1,139 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_SAMPLES = 1_000_000; +const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; + +function resolveArguments(argv) { + if ( + argv.length !== 4 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--output' || + argv[3].length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/summarize-samples.mjs --input --output ', + ); + } + return Object.freeze({ + inputPath: resolve(argv[1]), + outputDirectory: resolve(argv[3]), + }); +} + +function readBoundedJson(path) { + const bytes = readFileSync(path); + if (bytes.byteLength > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; +} + +function validateInput(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark sample input must be an object.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark sample contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + throw new Error('Benchmark unit is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES + ) { + throw new Error('Benchmark samples must be a non-empty bounded array.'); + } + if ( + value.samples.some( + (sample) => + typeof sample !== 'number' || !Number.isFinite(sample) || sample < 0, + ) + ) { + throw new Error('Benchmark samples must be finite non-negative numbers.'); + } + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + samples: Object.freeze([...value.samples]), + }); +} + +function nearestRank(sorted, percentile) { + const index = Math.ceil(percentile * sorted.length) - 1; + return sorted[index]; +} + +function summarize(input) { + const sorted = [...input.samples].sort((left, right) => left - right); + return Object.freeze({ + contractVersion: 1, + benchmarkId: input.benchmarkId, + unit: input.unit, + sampleCount: sorted.length, + percentileMethod: 'nearest-rank', + minimum: sorted[0], + p50: nearestRank(sorted, 0.5), + p75: nearestRank(sorted, 0.75), + p95: nearestRank(sorted, 0.95), + maximum: sorted.at(-1), + }); +} + +function formatSummary(summary) { + return [ + `benchmark=${summary.benchmarkId}`, + `unit=${summary.unit}`, + `samples=${summary.sampleCount}`, + `percentile_method=${summary.percentileMethod}`, + `minimum=${summary.minimum}`, + `p50=${summary.p50}`, + `p75=${summary.p75}`, + `p95=${summary.p95}`, + `maximum=${summary.maximum}`, + '', + ].join('\n'); +} + +function main() { + const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const input = validateInput(readBoundedJson(inputPath)); + const summary = summarize(input); + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync( + resolve(outputDirectory, 'summary.json'), + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8', + ); + writeFileSync( + resolve(outputDirectory, 'summary.txt'), + formatSummary(summary), + 'utf8', + ); +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark summary failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From 6a4dd666b579e6a25a473d0829fc9ef881fe1d71 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:11:48 -0700 Subject: [PATCH 016/260] test(perf): reject benchmark input output alias --- ...manceMeasurementStatisticsContract.test.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 6085aa25..e229bb8f 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -1,5 +1,6 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { + existsSync, mkdtempSync, readFileSync, rmSync, @@ -123,4 +124,32 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); -}); + + it('refuses to overwrite the measurement input with generated evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); + const input = join(root, 'summary.json'); + const output = root; + try { + writeInput(input, [10, 20, 30]); + const originalInput = readFileSync(input, 'utf8'); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(root, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file From a838dbefb3a410d9899985ae1f011e71af3a9c3d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:15:20 -0700 Subject: [PATCH 017/260] fix(perf): preserve benchmark sample inputs --- benchmarks/summarize-samples.mjs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index f695ae03..a20b0d2b 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -114,19 +114,20 @@ function formatSummary(summary) { function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const summaryJsonPath = resolve(outputDirectory, 'summary.json'); + const summaryTextPath = resolve(outputDirectory, 'summary.txt'); + if (inputPath === summaryJsonPath || inputPath === summaryTextPath) { + throw new Error('Benchmark output must not overwrite the sample input.'); + } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); mkdirSync(outputDirectory, { recursive: true }); writeFileSync( - resolve(outputDirectory, 'summary.json'), + summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8', ); - writeFileSync( - resolve(outputDirectory, 'summary.txt'), - formatSummary(summary), - 'utf8', - ); + writeFileSync(summaryTextPath, formatSummary(summary), 'utf8'); } try { From fe613959efed98bd4490192cfffbe44adeab8758 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:35:13 -0700 Subject: [PATCH 018/260] test(perf): reject hard-linked summary evidence aliases --- ...manceMeasurementStatisticsContract.test.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index e229bb8f..4e9853f6 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -1,6 +1,8 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { existsSync, + linkSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, @@ -152,4 +154,36 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); -}); \ No newline at end of file + + it('refuses a hard-linked output alias without mutating source evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(output, { recursive: true }); + linkSync(input, summaryJson); + const originalInput = readFileSync(input, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 33d5c45b1dbfa62b886371b9642c12d04687ad6b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:35:41 -0700 Subject: [PATCH 019/260] fix(perf): preserve hard-linked benchmark source evidence --- benchmarks/summarize-samples.mjs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index a20b0d2b..52306ccf 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,4 +1,10 @@ -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; @@ -112,16 +118,28 @@ function formatSummary(summary) { ].join('\n'); } +function refersToSameFile(leftPath, rightPath) { + if (!existsSync(leftPath) || !existsSync(rightPath)) return false; + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); const summaryTextPath = resolve(outputDirectory, 'summary.txt'); - if (inputPath === summaryJsonPath || inputPath === summaryTextPath) { + mkdirSync(outputDirectory, { recursive: true }); + if ( + inputPath === summaryJsonPath || + inputPath === summaryTextPath || + refersToSameFile(inputPath, summaryJsonPath) || + refersToSameFile(inputPath, summaryTextPath) + ) { throw new Error('Benchmark output must not overwrite the sample input.'); } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); - mkdirSync(outputDirectory, { recursive: true }); writeFileSync( summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, From 676555f04ec9955e127a9921700a1457c09b8ae1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:23:59 -0700 Subject: [PATCH 020/260] test(perf): prove oversized samples preflight whole-file reads --- ...manceMeasurementStatisticsContract.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 4e9853f6..d953ac03 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -6,10 +6,12 @@ import { mkdtempSync, readFileSync, rmSync, + truncateSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSummary { @@ -127,6 +129,50 @@ describe('deterministic benchmark sample statistics', () => { } }); + it('rejects obviously oversized sample input before whole-file reads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-size-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const preload = join(root, 'reject-whole-file-read.mjs'); + try { + writeFileSync(input, '', 'utf8'); + truncateSync(input, 16 * 1024 * 1024 + 1); + writeFileSync( + preload, + `import fs from 'node:fs';\nimport { syncBuiltinESMExports } from 'node:module';\nfs.readFileSync = () => { throw new Error('benchmark whole-file read sentinel'); };\nsyncBuiltinESMExports();\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + '--import', + pathToFileURL(preload).href, + script, + '--input', + input, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input exceeds the supported size.', + ); + expect(result.stderr).not.toContain('benchmark whole-file read sentinel'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('refuses to overwrite the measurement input with generated evidence', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); const input = join(root, 'summary.json'); From 72db249bd0ddc9d65fc2081c374573d0688d6bb2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:24:46 -0700 Subject: [PATCH 021/260] fix(perf): bound benchmark sample reads before allocation --- benchmarks/summarize-samples.mjs | 56 ++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 52306ccf..0daddcca 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,13 +1,17 @@ import { + closeSync, existsSync, + fstatSync, mkdirSync, - readFileSync, + openSync, + readSync, statSync, writeFileSync, } from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; @@ -31,17 +35,49 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const bytes = readFileSync(path); - if (bytes.byteLength > MAX_INPUT_BYTES) { - throw new Error('Benchmark sample input exceeds the supported size.'); - } - let parsed; + const descriptor = openSync(path, 'r'); try { - parsed = JSON.parse(bytes.toString('utf8')); - } catch { - throw new Error('Benchmark sample input must be valid JSON.'); + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + const bytes = Buffer.concat(chunks, totalBytes); + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; + } finally { + closeSync(descriptor); } - return parsed; } function validateInput(value) { From 9766bcfbdfeef398a179e5f05ae15ebc991406d9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:20:11 -0700 Subject: [PATCH 022/260] test(perf): require immutable benchmark provenance --- ...manceMeasurementStatisticsContract.test.ts | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index d953ac03..3c477543 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -18,6 +18,11 @@ interface BenchmarkSummary { readonly contractVersion: 1; readonly benchmarkId: string; readonly unit: string; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; readonly sampleCount: number; readonly percentileMethod: 'nearest-rank'; readonly minimum: number; @@ -28,6 +33,8 @@ interface BenchmarkSummary { } const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); function writeInput(path: string, samples: readonly number[]): void { writeFileSync( @@ -37,6 +44,11 @@ function writeInput(path: string, samples: readonly number[]): void { contractVersion: 1, benchmarkId: 'markdown-serialization-large', unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', samples, }, null, @@ -61,7 +73,7 @@ function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummar } describe('deterministic benchmark sample statistics', () => { - it('writes reproducible nearest-rank JSON and human-readable summaries', () => { + it('writes reproducible nearest-rank JSON and human-readable summaries with provenance metadata', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); const input = join(root, 'samples.json'); const first = join(root, 'first'); @@ -76,6 +88,11 @@ describe('deterministic benchmark sample statistics', () => { contractVersion: 1, benchmarkId: 'markdown-serialization-large', unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', sampleCount: 5, percentileMethod: 'nearest-rank', minimum: 10, @@ -88,6 +105,11 @@ describe('deterministic benchmark sample statistics', () => { const expectedText = [ 'benchmark=markdown-serialization-large', 'unit=ms', + `source_commit_sha=${SOURCE_COMMIT_SHA}`, + `artifact_sha256=${ARTIFACT_SHA256}`, + 'document_profile=large', + 'runtime_id=node-22.18.0', + 'reference_hardware_id=github-actions-ubuntu-24.04-x64', 'samples=5', 'percentile_method=nearest-rank', 'minimum=10', @@ -104,6 +126,42 @@ describe('deterministic benchmark sample statistics', () => { } }); + it('fails closed when immutable provenance metadata is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on invalid measurement samples without coercion', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); const input = join(root, 'samples.json'); From 738d3e7bc67905bed03fddc06af84f8f8c0cbd89 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:21:15 -0700 Subject: [PATCH 023/260] feat(perf): bind summaries to immutable provenance --- benchmarks/summarize-samples.mjs | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 0daddcca..1c82a1fc 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -15,6 +15,10 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); function resolveArguments(argv) { if ( @@ -96,6 +100,40 @@ function validateInput(value) { if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { throw new Error('Benchmark unit is invalid.'); } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error( + 'Benchmark artifactSha256 must be a lowercase 64-character SHA-256 digest.', + ); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark referenceHardwareId is invalid.'); + } if ( !Array.isArray(value.samples) || value.samples.length === 0 || @@ -114,6 +152,11 @@ function validateInput(value) { return Object.freeze({ benchmarkId: value.benchmarkId, unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, samples: Object.freeze([...value.samples]), }); } @@ -129,6 +172,11 @@ function summarize(input) { contractVersion: 1, benchmarkId: input.benchmarkId, unit: input.unit, + sourceCommitSha: input.sourceCommitSha, + artifactSha256: input.artifactSha256, + documentProfile: input.documentProfile, + runtimeId: input.runtimeId, + referenceHardwareId: input.referenceHardwareId, sampleCount: sorted.length, percentileMethod: 'nearest-rank', minimum: sorted[0], @@ -143,6 +191,11 @@ function formatSummary(summary) { return [ `benchmark=${summary.benchmarkId}`, `unit=${summary.unit}`, + `source_commit_sha=${summary.sourceCommitSha}`, + `artifact_sha256=${summary.artifactSha256}`, + `document_profile=${summary.documentProfile}`, + `runtime_id=${summary.runtimeId}`, + `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, `percentile_method=${summary.percentileMethod}`, `minimum=${summary.minimum}`, From 864fa8b9a6bd69312ca30b05dff1b7f6ae6ac4cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:33:13 -0700 Subject: [PATCH 024/260] test(perf): require strict UTF-8 benchmark evidence --- ...performanceMeasurementUtf8Contract.test.ts | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/performanceMeasurementUtf8Contract.test.ts diff --git a/src/performanceMeasurementUtf8Contract.test.ts b/src/performanceMeasurementUtf8Contract.test.ts new file mode 100644 index 00000000..45ceec9f --- /dev/null +++ b/src/performanceMeasurementUtf8Contract.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence UTF-8 contract', () => { + it('rejects malformed UTF-8 before parsing or generating summary evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-utf8-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + const prefix = [ + '{"contractVersion":1,', + '"benchmarkId":"markdown-serialization-large",', + '"unit":"ms",', + `"sourceCommitSha":"${SOURCE_COMMIT_SHA}",`, + `"artifactSha256":"${ARTIFACT_SHA256}",`, + '"documentProfile":"large",', + '"runtimeId":"node-22.18.0",', + '"referenceHardwareId":"github-actions-ubuntu-24.04-x64",', + '"samples":[1,2,3],', + '"untrustedNote":"', + ].join(''); + writeFileSync( + input, + Buffer.concat([ + Buffer.from(prefix, 'utf8'), + Buffer.from([0x80]), + Buffer.from('"}\n', 'utf8'), + ]), + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be valid UTF-8 JSON.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From bc887c0095fb2d97d13c86be3d4b8bf67cccd2cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:36:02 -0700 Subject: [PATCH 025/260] fix(perf): reject malformed UTF-8 benchmark evidence --- benchmarks/summarize-samples.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 1c82a1fc..39b6a139 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -19,6 +19,7 @@ const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); function resolveArguments(argv) { if ( @@ -72,9 +73,15 @@ function readBoundedJson(path) { } const bytes = Buffer.concat(chunks, totalBytes); + let text; + try { + text = UTF8_DECODER.decode(bytes); + } catch { + throw new Error('Benchmark sample input must be valid UTF-8 JSON.'); + } let parsed; try { - parsed = JSON.parse(bytes.toString('utf8')); + parsed = JSON.parse(text); } catch { throw new Error('Benchmark sample input must be valid JSON.'); } From 596c48e6055e0c717b09af05a4d40d84db34e7ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:43:41 -0700 Subject: [PATCH 026/260] test(perf): preflight invalid summary destinations --- ...rformanceMeasurementOutputContract.test.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/performanceMeasurementOutputContract.test.ts diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts new file mode 100644 index 00000000..aca7928e --- /dev/null +++ b/src/performanceMeasurementOutputContract.test.ts @@ -0,0 +1,61 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark summary output contract', () => { + it('rejects an invalid second destination before publishing the first summary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + mkdirSync(summaryText, { recursive: true }); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(summaryJson)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From f072e74983c70375af66f3718b71ff48b76ddc7d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:48:25 -0700 Subject: [PATCH 027/260] fix(perf): preflight benchmark output destinations --- benchmarks/summarize-samples.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 39b6a139..17fce6a7 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -2,6 +2,7 @@ import { closeSync, existsSync, fstatSync, + lstatSync, mkdirSync, openSync, readSync, @@ -221,6 +222,12 @@ function refersToSameFile(leftPath, rightPath) { return left.dev === right.dev && left.ino === right.ino; } +function assertRegularOutputDestination(path) { + if (existsSync(path) && !lstatSync(path).isFile()) { + throw new Error('Benchmark summary output paths must be regular files.'); + } +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); @@ -234,6 +241,8 @@ function main() { ) { throw new Error('Benchmark output must not overwrite the sample input.'); } + assertRegularOutputDestination(summaryJsonPath); + assertRegularOutputDestination(summaryTextPath); const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 3efb68cd7271226a42ece1ca5755d05b0a6ccb1b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:09:52 -0700 Subject: [PATCH 028/260] test(perf): reject arbitrary benchmark metadata --- ...formanceMeasurementPrivacyContract.test.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/performanceMeasurementPrivacyContract.test.ts diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts new file mode 100644 index 00000000..83dfccfd --- /dev/null +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -0,0 +1,59 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence privacy contract', () => { + it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + prompt: 'must-not-enter-benchmark-evidence', + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input contains unsupported fields.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 6cb64186e37fbfa1e1765fd8df31996915e9e849 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:13:41 -0700 Subject: [PATCH 029/260] fix(perf): reject unsupported benchmark metadata --- benchmarks/summarize-samples.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 17fce6a7..b457f71c 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -20,6 +20,17 @@ const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const BENCHMARK_INPUT_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'samples', +]); const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); function resolveArguments(argv) { @@ -96,6 +107,9 @@ function validateInput(value) { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Benchmark sample input must be an object.'); } + if (Object.keys(value).some((key) => !BENCHMARK_INPUT_KEYS.has(key))) { + throw new Error('Benchmark sample input contains unsupported fields.'); + } if (value.contractVersion !== 1) { throw new Error('Benchmark sample contractVersion must be 1.'); } From ba35b28f21b63ca8fbd8d2d5b8f2d0862464fa19 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:36:36 -0700 Subject: [PATCH 030/260] test(perf): require explicit benchmark regression comparator --- ...rmanceRegressionComparatorContract.test.ts | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 src/performanceRegressionComparatorContract.test.ts diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts new file mode 100644 index 00000000..0c7e72f5 --- /dev/null +++ b/src/performanceRegressionComparatorContract.test.ts @@ -0,0 +1,155 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type SummaryOverrides = Partial<{ + benchmarkId: string; + unit: string; + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + sampleCount: number; + percentileMethod: string; + minimum: number; + p50: number; + p75: number; + p95: number; + maximum: number; +}>; + +function summary(overrides: SummaryOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + ...overrides, + }; +} + +function runComparison( + root: string, + baseline: ReturnType, + current: ReturnType, + tolerancePercent: string, +) { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(baselinePath, `${JSON.stringify(baseline)}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(current)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + tolerancePercent, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('benchmark regression comparator contract', () => { + it('passes only when a current exact-context metric stays within an explicit tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); + try { + const result = runComparison(root, summary(), summary({ p95: 104 }), '5'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + metric: 'p95', + baselineValue: 100, + currentValue: 104, + maxRegressionPercent: 5, + regressionPercent: 4, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a material unapproved regression without hiding the measured receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); + try { + const result = runComparison(root, summary(), summary({ p95: 106 }), '5'); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + metric: 'p95', + baselineValue: 100, + currentValue: 106, + maxRegressionPercent: 5, + regressionPercent: 6, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects incomparable runtime or hardware evidence instead of laundering it through a tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-context-')); + try { + const result = runComparison( + root, + summary(), + summary({ referenceHardwareId: 'different-runner' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summaries are not comparable: referenceHardwareId differs.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires an explicit finite non-negative regression tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); + try { + const result = runComparison(root, summary(), summary(), '-1'); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 9504eadc784a68c32aad9e69b9806a97a25b5543 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:37:22 -0700 Subject: [PATCH 031/260] feat(perf): add explicit benchmark regression comparator --- benchmarks/compare-summaries.mjs | 286 +++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 benchmarks/compare-summaries.mjs diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs new file mode 100644 index 00000000..7c48fd77 --- /dev/null +++ b/benchmarks/compare-summaries.mjs @@ -0,0 +1,286 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); +const SUMMARY_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', + 'minimum', + 'p50', + 'p75', + 'p95', + 'maximum', +]); +const COMPARABLE_FIELDS = [ + 'benchmarkId', + 'unit', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', +]; +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 8 || + argv[0] !== '--baseline' || + argv[1].length === 0 || + argv[2] !== '--current' || + argv[3].length === 0 || + argv[4] !== '--metric' || + !METRICS.has(argv[5]) || + argv[6] !== '--max-regression-percent' || + argv[7].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/compare-summaries.mjs --baseline --current --metric --max-regression-percent ', + ); + } + + const maxRegressionPercent = Number(argv[7]); + if (!Number.isFinite(maxRegressionPercent) || maxRegressionPercent < 0) { + throw new Error( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } + + return Object.freeze({ + baselinePath: resolve(argv[1]), + currentPath: resolve(argv[3]), + metric: argv[5], + maxRegressionPercent, + }); +} + +function readBoundedJson(path) { + const descriptor = openSync(path, 'r'); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Benchmark summary input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Benchmark summary input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateSummary(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark summary input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== SUMMARY_KEYS.size || + keys.some((key) => !SUMMARY_KEYS.has(key)) + ) { + throw new Error('Benchmark summary input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark summary contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark summary benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + throw new Error('Benchmark summary unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Benchmark summary sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Benchmark summary artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark summary documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark summary runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark summary referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.sampleCount) || + value.sampleCount <= 0 || + value.sampleCount > MAX_SAMPLES + ) { + throw new Error('Benchmark summary sampleCount is invalid.'); + } + if (value.percentileMethod !== 'nearest-rank') { + throw new Error('Benchmark summary percentileMethod is invalid.'); + } + + const measurements = [ + value.minimum, + value.p50, + value.p75, + value.p95, + value.maximum, + ]; + if ( + measurements.some( + (measurement) => + typeof measurement !== 'number' || + !Number.isFinite(measurement) || + measurement < 0, + ) + ) { + throw new Error( + 'Benchmark summary measurements must be finite non-negative numbers.', + ); + } + for (let index = 1; index < measurements.length; index += 1) { + if (measurements[index] < measurements[index - 1]) { + throw new Error('Benchmark summary percentile ordering is invalid.'); + } + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + sampleCount: value.sampleCount, + percentileMethod: value.percentileMethod, + minimum: value.minimum, + p50: value.p50, + p75: value.p75, + p95: value.p95, + maximum: value.maximum, + }); +} + +function assertComparable(baseline, current) { + for (const field of COMPARABLE_FIELDS) { + if (baseline[field] !== current[field]) { + throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); + } + } +} + +function normalizePercent(value) { + const rounded = Number(value.toFixed(12)); + return Object.is(rounded, -0) ? 0 : rounded; +} + +function compare(baseline, current, metric, maxRegressionPercent) { + assertComparable(baseline, current); + const baselineValue = baseline[metric]; + const currentValue = current[metric]; + if (baselineValue === 0 && currentValue !== 0) { + throw new Error( + 'Benchmark regression percent is undefined for a zero non-matching baseline.', + ); + } + const regressionPercent = + baselineValue === 0 + ? 0 + : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + return Object.freeze({ + contractVersion: 1, + benchmarkId: baseline.benchmarkId, + metric, + baselineValue, + currentValue, + maxRegressionPercent, + regressionPercent, + passed: regressionPercent <= maxRegressionPercent, + }); +} + +function main() { + const { baselinePath, currentPath, metric, maxRegressionPercent } = + resolveArguments(process.argv.slice(2)); + const baseline = validateSummary(readBoundedJson(baselinePath)); + const current = validateSummary(readBoundedJson(currentPath)); + const result = compare(baseline, current, metric, maxRegressionPercent); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark comparison failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From f5439a1f0708182526449e98d6fcbab7feaedb7f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:40:00 -0700 Subject: [PATCH 032/260] test(perf): prove comparator rejects blocking FIFO inputs --- ...rmanceRegressionComparatorContract.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 0c7e72f5..7392cd03 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -152,4 +152,42 @@ describe('benchmark regression comparator contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fifo-')); + const baselinePath = join(root, 'baseline.pipe'); + const currentPath = join(root, 'current.json'); + try { + const mkfifo = spawnSync('mkfifo', [baselinePath], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + writeFileSync(currentPath, `${JSON.stringify(summary())}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8', timeout: 1000 }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 13c09c58dcb10849b6bdac756bfd6a53da8ae2d4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:40:19 -0700 Subject: [PATCH 033/260] test(perf): prove summarizer rejects blocking FIFO inputs --- ...rformanceMeasurementOutputContract.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index aca7928e..6ae5fc90 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -58,4 +58,37 @@ describe('benchmark summary output contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-input-fifo-')); + const input = join(root, 'samples.pipe'); + const output = join(root, 'output'); + try { + const mkfifo = spawnSync('mkfifo', [input], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 1000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From dbf53455d9cc2d76d41c23d64223da8942a19f01 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:41:04 -0700 Subject: [PATCH 034/260] fix(perf): validate comparator inputs without blocking --- benchmarks/compare-summaries.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 7c48fd77..15f60255 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -1,9 +1,17 @@ -import { closeSync, fstatSync, openSync, readSync } from 'node:fs'; +import { + closeSync, + constants, + fstatSync, + openSync, + readSync, +} from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -72,7 +80,7 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, 'r'); + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From e98faeeaad97aa64098efbe75178acfbd2066634 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:41:36 -0700 Subject: [PATCH 035/260] fix(perf): validate sample inputs without blocking --- benchmarks/summarize-samples.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index b457f71c..ee7de3e3 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,5 +1,6 @@ import { closeSync, + constants, existsSync, fstatSync, lstatSync, @@ -14,6 +15,8 @@ import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -52,7 +55,7 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, 'r'); + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From d82015018dea0ca03fc1777a32e41af1161faf96 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:57:39 -0700 Subject: [PATCH 036/260] test(perf): reject dangling benchmark output symlinks --- ...rformanceMeasurementOutputContract.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index 6ae5fc90..ade9f61d 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -4,6 +4,7 @@ import { mkdirSync, mkdtempSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -59,6 +60,53 @@ describe('benchmark summary output contract', () => { } }); + it('rejects a dangling summary symlink before it can create the symlink target', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-symlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const escapedTarget = join(root, 'escaped-summary.json'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + mkdirSync(output, { recursive: true }); + symlinkSync(escapedTarget, summaryJson); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(escapedTarget)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 42c951670b3482a8ec36370deb660a9a83a544ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:59:19 -0700 Subject: [PATCH 037/260] fix(perf): reject dangling benchmark output symlinks --- benchmarks/summarize-samples.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index ee7de3e3..d857865f 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -240,7 +240,8 @@ function refersToSameFile(leftPath, rightPath) { } function assertRegularOutputDestination(path) { - if (existsSync(path) && !lstatSync(path).isFile()) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } } From 2b2f537fda7729d374b4f33ff74caf68979367d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:08:55 -0700 Subject: [PATCH 038/260] test(perf): fail closed on regression percentage overflow --- ...formanceRegressionOverflowContract.test.ts | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/performanceRegressionOverflowContract.test.ts diff --git a/src/performanceRegressionOverflowContract.test.ts b/src/performanceRegressionOverflowContract.test.ts new file mode 100644 index 00000000..2676c458 --- /dev/null +++ b/src/performanceRegressionOverflowContract.test.ts @@ -0,0 +1,71 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(measurement: number, digestCharacter: string) { + return { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: digestCharacter.repeat(40), + artifactSha256: digestCharacter.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 1, + percentileMethod: 'nearest-rank', + minimum: measurement, + p50: measurement, + p75: measurement, + p95: measurement, + maximum: measurement, + }; +} + +describe('benchmark regression comparator overflow contract', () => { + it('fails closed instead of serializing an overflowing regression percentage as JSON null', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-overflow-')); + const baseline = join(root, 'baseline.json'); + const current = join(root, 'current.json'); + try { + writeFileSync( + baseline, + `${JSON.stringify(summary(1e-308, 'a'))}\n`, + 'utf8', + ); + writeFileSync( + current, + `${JSON.stringify(summary(1e308, 'b'))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baseline, + '--current', + current, + '--metric', + 'p95', + '--max-regression-percent', + '10', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 712cd9db034d0c6cf2bc5e1e631c62d14d256a7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:09:31 -0700 Subject: [PATCH 039/260] fix(perf): fail closed on regression percentage overflow --- benchmarks/compare-summaries.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 15f60255..a2c3ea62 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -262,6 +262,11 @@ function compare(baseline, current, metric, maxRegressionPercent) { baselineValue === 0 ? 0 : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + if (!Number.isFinite(regressionPercent)) { + throw new Error( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } return Object.freeze({ contractVersion: 1, benchmarkId: baseline.benchmarkId, From 1604f20475b80df255320e71fe0d19e551f67a07 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:06:34 -0700 Subject: [PATCH 040/260] test(perf): require provenance-bound regression receipts --- ...rmanceRegressionProvenanceContract.test.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/performanceRegressionProvenanceContract.test.ts diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts new file mode 100644 index 00000000..48d55840 --- /dev/null +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -0,0 +1,92 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(sourceCommitSha: string, artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha, + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression provenance contract', () => { + it('binds each comparison receipt to both exact measured artifacts and the shared context', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const baselineSourceCommitSha = 'a'.repeat(40); + const currentSourceCommitSha = 'c'.repeat(40); + const baselineArtifactSha256 = 'b'.repeat(64); + const currentArtifactSha256 = 'd'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary(baselineSourceCommitSha, baselineArtifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary(currentSourceCommitSha, currentArtifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha, + baselineArtifactSha256, + currentSourceCommitSha, + currentArtifactSha256, + baselineValue: 100, + currentValue: 100, + maxRegressionPercent: 5, + regressionPercent: 0, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From ceac649067a35dfdf3567256828c23de7d950aa5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:07:35 -0700 Subject: [PATCH 041/260] fix(perf): bind regression receipts to exact evidence --- benchmarks/compare-summaries.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index a2c3ea62..b41471c1 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -270,7 +270,17 @@ function compare(baseline, current, metric, maxRegressionPercent) { return Object.freeze({ contractVersion: 1, benchmarkId: baseline.benchmarkId, + unit: baseline.unit, + documentProfile: baseline.documentProfile, + runtimeId: baseline.runtimeId, + referenceHardwareId: baseline.referenceHardwareId, + sampleCount: baseline.sampleCount, + percentileMethod: baseline.percentileMethod, metric, + baselineSourceCommitSha: baseline.sourceCommitSha, + baselineArtifactSha256: baseline.artifactSha256, + currentSourceCommitSha: current.sourceCommitSha, + currentArtifactSha256: current.artifactSha256, baselineValue, currentValue, maxRegressionPercent, From 4dc7984ab131188d59fbba7844789168e63e0393 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:09:42 -0700 Subject: [PATCH 042/260] test(perf): align comparator receipt contract --- ...rmanceRegressionComparatorContract.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 7392cd03..57fcab69 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -84,7 +84,17 @@ describe('benchmark regression comparator contract', () => { expect(JSON.parse(result.stdout)).toEqual({ contractVersion: 1, benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: ARTIFACT_SHA256, baselineValue: 100, currentValue: 104, maxRegressionPercent: 5, @@ -106,7 +116,17 @@ describe('benchmark regression comparator contract', () => { expect(JSON.parse(result.stdout)).toEqual({ contractVersion: 1, benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: ARTIFACT_SHA256, baselineValue: 100, currentValue: 106, maxRegressionPercent: 5, From 36af472d144dd961a126375ba427350ffc30f167 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 15:34:24 -0700 Subject: [PATCH 043/260] test(perf): reject aliased summary destinations --- ...rformanceMeasurementOutputContract.test.ts | 85 ++++++++++++------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index ade9f61d..45376b1f 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -1,8 +1,10 @@ import { spawnSync } from 'node:child_process'; import { existsSync, + linkSync, mkdirSync, mkdtempSync, + readFileSync, rmSync, symlinkSync, writeFileSync, @@ -15,6 +17,24 @@ const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +function writeValidInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + describe('benchmark summary output contract', () => { it('rejects an invalid second destination before publishing the first summary', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); @@ -23,21 +43,7 @@ describe('benchmark summary output contract', () => { const summaryJson = join(output, 'summary.json'); const summaryText = join(output, 'summary.txt'); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [1, 2, 3], - })}\n`, - 'utf8', - ); + writeValidInput(input); mkdirSync(summaryText, { recursive: true }); const result = spawnSync( @@ -69,21 +75,7 @@ describe('benchmark summary output contract', () => { const summaryJson = join(output, 'summary.json'); const escapedTarget = join(root, 'escaped-summary.json'); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [1, 2, 3], - })}\n`, - 'utf8', - ); + writeValidInput(input); mkdirSync(output, { recursive: true }); symlinkSync(escapedTarget, summaryJson); @@ -107,6 +99,39 @@ describe('benchmark summary output contract', () => { } }); + it('rejects two output paths that alias the same regular file before publication', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-alias-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + writeFileSync(summaryJson, 'sentinel', 'utf8'); + linkSync(summaryJson, summaryText); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary outputs must be distinct files.', + ); + expect(readFileSync(summaryJson, 'utf8')).toBe('sentinel'); + expect(readFileSync(summaryText, 'utf8')).toBe('sentinel'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From a86955632a201f7592ae00c81f1f3e3b8148b8ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 15:35:10 -0700 Subject: [PATCH 044/260] fix(perf): reject aliased summary outputs --- benchmarks/summarize-samples.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index d857865f..12f8cc7b 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -261,6 +261,9 @@ function main() { } assertRegularOutputDestination(summaryJsonPath); assertRegularOutputDestination(summaryTextPath); + if (refersToSameFile(summaryJsonPath, summaryTextPath)) { + throw new Error('Benchmark summary outputs must be distinct files.'); + } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 1902b63f92ecaa2ef03250c2a758edde2099dc2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 16:57:43 -0700 Subject: [PATCH 045/260] test(perf): reject vacuous same-artifact comparisons --- ...rmanceRegressionProvenanceContract.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts index 48d55840..55e90cda 100644 --- a/src/performanceRegressionProvenanceContract.test.ts +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -89,4 +89,47 @@ describe('benchmark regression provenance contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('rejects a vacuous comparison when both summaries identify the same exact artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const artifactSha256 = 'b'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary('a'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary('c'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark summaries must identify distinct measured artifacts.\n', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From a7467876ae37ad70965536b8aa15cc23cbd52a8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:00:12 -0700 Subject: [PATCH 046/260] fix(perf): reject same-artifact regression evidence --- benchmarks/compare-summaries.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index b41471c1..af21bb45 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -242,6 +242,11 @@ function assertComparable(baseline, current) { throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); } } + if (baseline.artifactSha256 === current.artifactSha256) { + throw new Error( + 'Benchmark summaries must identify distinct measured artifacts.', + ); + } } function normalizePercent(value) { From 8e321140ef4beeeaf247f46958b38f02eebfff45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:02:19 -0700 Subject: [PATCH 047/260] test(perf): use distinct artifacts in comparator receipts --- ...rmanceRegressionComparatorContract.test.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 57fcab69..4b0a556d 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from 'vitest'; const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +const CURRENT_ARTIFACT_SHA256 = 'c'.repeat(64); type SummaryOverrides = Partial<{ benchmarkId: string; @@ -77,7 +78,12 @@ describe('benchmark regression comparator contract', () => { it('passes only when a current exact-context metric stays within an explicit tolerance', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); try { - const result = runComparison(root, summary(), summary({ p95: 104 }), '5'); + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + '5', + ); expect(result.status).toBe(0); expect(result.stderr).toBe(''); @@ -94,7 +100,7 @@ describe('benchmark regression comparator contract', () => { baselineSourceCommitSha: SOURCE_COMMIT_SHA, baselineArtifactSha256: ARTIFACT_SHA256, currentSourceCommitSha: SOURCE_COMMIT_SHA, - currentArtifactSha256: ARTIFACT_SHA256, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, baselineValue: 100, currentValue: 104, maxRegressionPercent: 5, @@ -109,7 +115,12 @@ describe('benchmark regression comparator contract', () => { it('fails a material unapproved regression without hiding the measured receipt', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); try { - const result = runComparison(root, summary(), summary({ p95: 106 }), '5'); + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 106 }), + '5', + ); expect(result.status).toBe(1); expect(result.stderr).toBe(''); @@ -126,7 +137,7 @@ describe('benchmark regression comparator contract', () => { baselineSourceCommitSha: SOURCE_COMMIT_SHA, baselineArtifactSha256: ARTIFACT_SHA256, currentSourceCommitSha: SOURCE_COMMIT_SHA, - currentArtifactSha256: ARTIFACT_SHA256, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, baselineValue: 100, currentValue: 106, maxRegressionPercent: 5, From f6b87db8c3ee82cf37a3f3e0bfdfa4472a7bac9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:30:44 -0700 Subject: [PATCH 048/260] test(perf): reject private-looking benchmark evidence identifiers --- ...formanceMeasurementPrivacyContract.test.ts | 94 +++++++++++++------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts index 83dfccfd..1d7a3c5b 100644 --- a/src/performanceMeasurementPrivacyContract.test.ts +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -13,38 +13,43 @@ const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +const validInput = { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], +} as const; + +function runSummary(inputValue: object) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + writeFileSync(input, `${JSON.stringify(inputValue)}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + describe('benchmark evidence privacy contract', () => { it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { - const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); - const input = join(root, 'samples.json'); - const output = join(root, 'output'); + const { root, output, result } = runSummary({ + ...validInput, + prompt: 'must-not-enter-benchmark-evidence', + }); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [10, 20, 30], - prompt: 'must-not-enter-benchmark-evidence', - })}\n`, - 'utf8', - ); - - const result = spawnSync( - process.execPath, - [script, '--input', input, '--output', output], - { - cwd: process.cwd(), - encoding: 'utf8', - }, - ); - expect(result.status).toBe(1); expect(result.stdout).toBe(''); expect(result.stderr.trim()).toBe( @@ -56,4 +61,35 @@ describe('benchmark evidence privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it.each([ + [ + 'benchmarkId', + 'tenant-acme-large', + 'Benchmark benchmarkId is invalid.', + ], + ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], + [ + 'referenceHardwareId', + 'tenant-acme', + 'Benchmark referenceHardwareId is invalid.', + ], + ])( + 'rejects caller-controlled %s values that could launder private identifiers into evidence', + (field, value, expectedError) => { + const { root, output, result } = runSummary({ + ...validInput, + [field]: value, + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedError); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); }); From b6fef32bcbc57d27ec3478d7b995bf13c3e7b073 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:36:01 -0700 Subject: [PATCH 049/260] test(perf): keep hardware mismatch fixture privacy-safe --- src/performanceRegressionComparatorContract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 4b0a556d..fd9806ee 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -155,7 +155,7 @@ describe('benchmark regression comparator contract', () => { const result = runComparison( root, summary(), - summary({ referenceHardwareId: 'different-runner' }), + summary({ referenceHardwareId: 'github-actions-ubuntu-22.04-x64' }), '5', ); From 4a7786cc780406bdeac5be1206f09b87436d809a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:36:42 -0700 Subject: [PATCH 050/260] fix(perf): constrain benchmark evidence identifiers --- benchmarks/summarize-samples.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 12f8cc7b..6daa81f5 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -17,11 +17,15 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); -const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); const BENCHMARK_INPUT_KEYS = new Set([ 'contractVersion', @@ -149,13 +153,13 @@ function validateInput(value) { } if ( typeof value.runtimeId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.runtimeId) + !RUNTIME_ID_PATTERN.test(value.runtimeId) ) { throw new Error('Benchmark runtimeId is invalid.'); } if ( typeof value.referenceHardwareId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) ) { throw new Error('Benchmark referenceHardwareId is invalid.'); } From 37ec73e7cdb34d58ffc86d7664b5ddba66a99ca0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:37:40 -0700 Subject: [PATCH 051/260] fix(perf): validate comparison evidence identifiers --- benchmarks/compare-summaries.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index af21bb45..65f4ea37 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -12,11 +12,15 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); -const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); const SUMMARY_KEYS = new Set([ @@ -172,13 +176,13 @@ function validateSummary(value) { } if ( typeof value.runtimeId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.runtimeId) + !RUNTIME_ID_PATTERN.test(value.runtimeId) ) { throw new Error('Benchmark summary runtimeId is invalid.'); } if ( typeof value.referenceHardwareId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) ) { throw new Error('Benchmark summary referenceHardwareId is invalid.'); } From 142499dee4585dfdd5a307d287d38497e6604fbb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:42:43 -0700 Subject: [PATCH 052/260] test(perf): reject private-looking benchmark units --- src/performanceMeasurementPrivacyContract.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts index 1d7a3c5b..9f994c5e 100644 --- a/src/performanceMeasurementPrivacyContract.test.ts +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -68,6 +68,7 @@ describe('benchmark evidence privacy contract', () => { 'tenant-acme-large', 'Benchmark benchmarkId is invalid.', ], + ['unit', 'tenant-acme', 'Benchmark unit is invalid.'], ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], [ 'referenceHardwareId', From 7248c18c57ce8da60be82beb4cc3086aa1dda358 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:45:16 -0700 Subject: [PATCH 053/260] test(perf): reject private units at comparator boundary --- ...ormanceRegressionComparatorContract.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index fd9806ee..997a02ac 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -169,6 +169,24 @@ describe('benchmark regression comparator contract', () => { } }); + it('rejects private-looking units at the direct summary-comparison boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-unit-')); + try { + const result = runComparison( + root, + summary({ unit: 'tenant-acme' }), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, unit: 'tenant-acme' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summary unit is invalid.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('requires an explicit finite non-negative regression tolerance', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); try { From 92eaeb13c2c3d71dd185751ad941f397fbbd73c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:49:26 -0700 Subject: [PATCH 054/260] fix(perf): constrain benchmark measurement units --- benchmarks/summarize-samples.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 6daa81f5..354c2539 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -19,7 +19,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; -const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -126,7 +126,7 @@ function validateInput(value) { ) { throw new Error('Benchmark benchmarkId is invalid.'); } - if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark unit is invalid.'); } if ( From 3fe89a3dfde92b592b1ddb4bbeb0e5012255e7ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:50:06 -0700 Subject: [PATCH 055/260] fix(perf): validate comparator measurement units --- benchmarks/compare-summaries.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 65f4ea37..975e8c97 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -14,7 +14,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; -const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -153,7 +153,7 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary benchmarkId is invalid.'); } - if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark summary unit is invalid.'); } if ( From 3e1f34585885a7f7a52a8c110f306dd6207cc5f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:34:49 -0700 Subject: [PATCH 056/260] test(perf): require markdown measurement harness --- ...ormanceMarkdownMeasurementContract.test.ts | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 src/performanceMarkdownMeasurementContract.test.ts diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts new file mode 100644 index 00000000..fb763477 --- /dev/null +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -0,0 +1,211 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); +const RUNTIME_ID = 'node-22.18.0'; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function measurementArguments( + input: string, + modulePath: string, + output: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + ARTIFACT_SHA256, + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +describe('Markdown runtime measurement contract', () => { + it('writes bounded privacy-safe samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeFileSync(input, '# Buyer benchmark fixture\n\nSynthetic content only.\n', 'utf8'); + writeFileSync( + modulePath, + "export function markdownToHtml(source) { return `

${source.length}

`; }\n", + 'utf8', + ); + + execFileSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Buyer benchmark fixture'); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Synthetic content only'); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const summary = JSON.parse( + readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), + ) as { sampleCount: number; benchmarkId: string; unit: string }; + expect(summary).toMatchObject({ + sampleCount: 3, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the public serializer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must export markdownToHtml().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked document inputs before reading benchmark content', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-symlink-')); + const realInput = join(root, 'real.md'); + const input = join(root, 'linked.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(realInput, '# Synthetic\n', 'utf8'); + symlinkSync(realInput, input); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return source; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires a file-backed module URL rather than network or package-name resolution', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); + const input = join(root, 'small.md'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments( + input, + 'https://example.invalid/markdown.mjs', + samplesPath, + ), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(samplesPath)).toBe(false); + expect(() => pathToFileURL(input)).not.toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 124e9446ad3799ee4eba479d486010fa562570ef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:35:59 -0700 Subject: [PATCH 057/260] feat(perf): add bounded markdown runtime measurement --- benchmarks/measure-markdown.mjs | 250 ++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 benchmarks/measure-markdown.mjs diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs new file mode 100644 index 00000000..e1728b1b --- /dev/null +++ b/benchmarks/measure-markdown.mjs @@ -0,0 +1,250 @@ +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { mkdirSync } from 'node:fs'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; + +function resolveArguments(argv) { + const expectedFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Markdown benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Markdown benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Markdown benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Markdown benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Markdown benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Markdown benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function readBoundedMarkdown(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink() || !pathMetadata.isFile()) { + throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + } + + const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Markdown benchmark input exceeds the supported size.'); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Markdown benchmark input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + try { + return new TextDecoder('utf-8', { fatal: true }).decode( + Buffer.concat(chunks, totalBytes), + ); + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } + } finally { + closeSync(descriptor); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let modulePath; + try { + modulePath = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if (modulePath.protocol !== 'file:') { + throw new Error('Measured Markdown module must be a local regular file.'); + } + const resolvedPath = resolve(decodeURIComponent(modulePath.pathname)); + const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + if (metadata === undefined || metadata.isSymbolicLink() || !metadata.isFile()) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + return realpathSync(resolvedPath); +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + if (args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath)) { + throw new Error('Markdown benchmark output must not overwrite its input.'); + } + const source = readBoundedMarkdown(args.inputPath); + const modulePath = resolveLocalModule(args.modulePath); + const measuredModule = await import(pathToFileURL(modulePath).href); + if (typeof measuredModule.markdownToHtml !== 'function') { + throw new Error('Measured Markdown module must export markdownToHtml().'); + } + + const warmup = measuredModule.markdownToHtml(source); + if (typeof warmup !== 'string') { + throw new Error('Measured markdownToHtml() must return a string.'); + } + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + const output = measuredModule.markdownToHtml(source); + const elapsed = performance.now() - start; + if (typeof output !== 'string' || !Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Markdown measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + mkdirSync(dirname(args.outputPath), { recursive: true }); + writeFileSync( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `markdown-serialization-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Markdown benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From a08eb096b907ab9c4c0a2b44b39496fc547bf80d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:37:52 -0700 Subject: [PATCH 058/260] test(perf): bind markdown measurements to measured artifact --- ...kdownMeasurementProvenanceContract.test.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/performanceMarkdownMeasurementProvenanceContract.test.ts diff --git a/src/performanceMarkdownMeasurementProvenanceContract.test.ts b/src/performanceMarkdownMeasurementProvenanceContract.test.ts new file mode 100644 index 00000000..dea6bb44 --- /dev/null +++ b/src/performanceMarkdownMeasurementProvenanceContract.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement artifact provenance', () => { + it('rejects caller metadata that does not match the exact measured module bytes', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-provenance-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'f'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From e29e5698e7133837592e7ec65dc2452723350bb5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:39:11 -0700 Subject: [PATCH 059/260] test(perf): use exact measured module digests --- ...rformanceMarkdownMeasurementContract.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index fb763477..54d4fd8a 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -1,4 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { existsSync, mkdtempSync, @@ -30,14 +31,21 @@ const measurementScript = resolve( ); const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const ARTIFACT_SHA256 = 'b'.repeat(64); +const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); const RUNTIME_ID = 'node-22.18.0'; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + function measurementArguments( input: string, modulePath: string, output: string, + artifactSha256 = existsSync(modulePath) + ? fileSha256(modulePath) + : FALLBACK_ARTIFACT_SHA256, ): string[] { return [ measurementScript, @@ -52,7 +60,7 @@ function measurementArguments( '--source-commit-sha', SOURCE_COMMIT_SHA, '--artifact-sha256', - ARTIFACT_SHA256, + artifactSha256, '--runtime-id', RUNTIME_ID, '--reference-hardware-id', @@ -76,10 +84,11 @@ describe('Markdown runtime measurement contract', () => { "export function markdownToHtml(source) { return `

${source.length}

`; }\n", 'utf8', ); + const artifactSha256 = fileSha256(modulePath); execFileSync( process.execPath, - measurementArguments(input, modulePath, samplesPath), + measurementArguments(input, modulePath, samplesPath, artifactSha256), { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, ); @@ -91,7 +100,7 @@ describe('Markdown runtime measurement contract', () => { benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, + artifactSha256, documentProfile: 'large', runtimeId: RUNTIME_ID, referenceHardwareId: HARDWARE_ID, From 866ca00cc73dd68eecddec18903f5accd0b9d8af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:39:42 -0700 Subject: [PATCH 060/260] fix(perf): verify measured markdown artifact provenance --- benchmarks/measure-markdown.mjs | 101 +++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e1728b1b..d8713a49 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -1,9 +1,11 @@ +import { createHash } from 'node:crypto'; import { performance } from 'node:perf_hooks'; import { closeSync, constants, fstatSync, lstatSync, + mkdirSync, openSync, readSync, realpathSync, @@ -12,9 +14,9 @@ import { } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { mkdirSync } from 'node:fs'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = @@ -98,25 +100,29 @@ function resolveArguments(argv) { }); } -function readBoundedMarkdown(path) { +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - if (pathMetadata === undefined || pathMetadata.isSymbolicLink() || !pathMetadata.isFile()) { - throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); } const descriptor = openSync(path, READ_ONLY_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + throw new Error(invalidFileMessage); } - if (metadata.size > MAX_INPUT_BYTES) { - throw new Error('Markdown benchmark input exceeds the supported size.'); + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); } const chunks = []; let totalBytes = 0; - while (totalBytes <= MAX_INPUT_BYTES) { - const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; const chunk = Buffer.allocUnsafe( Math.min(READ_CHUNK_BYTES, remainingBudget), ); @@ -129,24 +135,31 @@ function readBoundedMarkdown(path) { ); if (bytesRead === 0) break; totalBytes += bytesRead; - if (totalBytes > MAX_INPUT_BYTES) { - throw new Error('Markdown benchmark input exceeds the supported size.'); + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); } chunks.push(chunk.subarray(0, bytesRead)); } - - try { - return new TextDecoder('utf-8', { fatal: true }).decode( - Buffer.concat(chunks, totalBytes), - ); - } catch { - throw new Error('Markdown benchmark input must be valid UTF-8.'); - } + return Buffer.concat(chunks, totalBytes); } finally { closeSync(descriptor); } } +function readBoundedMarkdown(path) { + const bytes = readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Markdown benchmark input must be a regular non-symlink file.', + 'Markdown benchmark input exceeds the supported size.', + ); + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } +} + function resolveLocalModule(pathOrUrl) { if ( pathOrUrl.startsWith('http:') || @@ -156,25 +169,47 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured Markdown module must be a local regular file.'); } - let modulePath; + let moduleUrl; try { - modulePath = pathOrUrl.startsWith('file:') + moduleUrl = pathOrUrl.startsWith('file:') ? new URL(pathOrUrl) : pathToFileURL(resolve(pathOrUrl)); } catch { throw new Error('Measured Markdown module must be a local regular file.'); } - if (modulePath.protocol !== 'file:') { + if (moduleUrl.protocol !== 'file:') { throw new Error('Measured Markdown module must be a local regular file.'); } - const resolvedPath = resolve(decodeURIComponent(modulePath.pathname)); + const resolvedPath = resolve(decodeURIComponent(moduleUrl.pathname)); const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); - if (metadata === undefined || metadata.isSymbolicLink() || !metadata.isFile()) { + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { throw new Error('Measured Markdown module must be a local regular file.'); } return realpathSync(resolvedPath); } +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured Markdown module must be a local regular file.', + 'Measured Markdown module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); if (rightMetadata === undefined) return false; @@ -188,11 +223,16 @@ function refersToSameFile(leftPath, rightPath) { async function main() { const args = resolveArguments(process.argv.slice(2)); - if (args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath)) { + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { throw new Error('Markdown benchmark output must not overwrite its input.'); } const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + const measuredModule = await import(pathToFileURL(modulePath).href); if (typeof measuredModule.markdownToHtml !== 'function') { throw new Error('Measured Markdown module must export markdownToHtml().'); @@ -208,12 +248,17 @@ async function main() { const start = performance.now(); const output = measuredModule.markdownToHtml(source); const elapsed = performance.now() - start; - if (typeof output !== 'string' || !Number.isFinite(elapsed) || elapsed < 0) { + if ( + typeof output !== 'string' || + !Number.isFinite(elapsed) || + elapsed < 0 + ) { throw new Error('Markdown measurement produced invalid runtime evidence.'); } samples.push(elapsed); } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); @@ -244,7 +289,9 @@ try { await main(); } catch (error) { const message = - error instanceof Error ? error.message : 'Markdown benchmark measurement failed.'; + error instanceof Error + ? error.message + : 'Markdown benchmark measurement failed.'; process.stderr.write(`${message}\n`); process.exitCode = 1; } From 1ecdba32613fe706e7f5fa27154972b5bafef2fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:42:13 -0700 Subject: [PATCH 061/260] test(perf): reject measurement output aliasing module --- ...ormanceMarkdownMeasurementContract.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 54d4fd8a..87edd0e3 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -190,6 +190,32 @@ describe('Markdown runtime measurement contract', () => { } }); + it('rejects an output path that aliases the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-output-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, modulePath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output must not overwrite the measured module.', + ); + expect(readFileSync(modulePath, 'utf8')).toBe(moduleSource); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('requires a file-backed module URL rather than network or package-name resolution', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); const input = join(root, 'small.md'); From b5373c9d51d326e7b5e73446ba7445a17704c185 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:42:47 -0700 Subject: [PATCH 062/260] fix(perf): preserve measured module from output writes --- benchmarks/measure-markdown.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index d8713a49..2154089a 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -231,6 +231,14 @@ async function main() { } const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Markdown benchmark output must not overwrite the measured module.', + ); + } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const measuredModule = await import(pathToFileURL(modulePath).href); From 4fb4f3faed55430bb777535596625054c4bbcecb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:48:30 -0700 Subject: [PATCH 063/260] test(perf): reject non-local file URL authorities --- ...rformanceMarkdownModuleUrlContract.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/performanceMarkdownModuleUrlContract.test.ts diff --git a/src/performanceMarkdownModuleUrlContract.test.ts b/src/performanceMarkdownModuleUrlContract.test.ts new file mode 100644 index 00000000..af022d47 --- /dev/null +++ b/src/performanceMarkdownModuleUrlContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement module URL authority', () => { + it('rejects file URLs with a non-local host before loading the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-url-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + const artifactSha256 = createHash('sha256') + .update(readFileSync(modulePath)) + .digest('hex'); + const nonLocalFileUrl = pathToFileURL(modulePath); + nonLocalFileUrl.hostname = 'example.invalid'; + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + nonLocalFileUrl.href, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 2bcd1de9569f51bef83be2d0ae4f13f2cf085f60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:49:22 -0700 Subject: [PATCH 064/260] fix(perf): require local file URL authority --- benchmarks/measure-markdown.mjs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 2154089a..04357926 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -13,7 +13,7 @@ import { writeFileSync, } from 'node:fs'; import { dirname, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; @@ -180,7 +180,12 @@ function resolveLocalModule(pathOrUrl) { if (moduleUrl.protocol !== 'file:') { throw new Error('Measured Markdown module must be a local regular file.'); } - const resolvedPath = resolve(decodeURIComponent(moduleUrl.pathname)); + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); if ( metadata === undefined || From 398c16b5dc6aaafda5296470908ed8960cbac8bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:36:26 -0700 Subject: [PATCH 065/260] test(perf): redact measured module failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts new file mode 100644 index 00000000..4f6a08c3 --- /dev/null +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -0,0 +1,96 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function runMeasurement(moduleSource: string) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('Markdown measurement error privacy contract', () => { + it('redacts exceptions raised while loading the measured module', () => { + const privateSentinel = 'private-import-sentinel-must-not-leak'; + const moduleSource = `throw new Error('${privateSentinel}');\nexport function markdownToHtml(value) { return value; }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module could not be loaded.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts exceptions raised by the measured serializer', () => { + const privateSentinel = 'private-serializer-sentinel-must-not-leak'; + const moduleSource = `export function markdownToHtml() { throw new Error('${privateSentinel}'); }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured markdownToHtml() execution failed.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 8369d42a1ae18fd2e6096e574e456f9d08209b78 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:39:12 -0700 Subject: [PATCH 066/260] fix(perf): redact measured module failures --- benchmarks/measure-markdown.mjs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 04357926..c50733c7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -226,6 +226,22 @@ function refersToSameFile(leftPath, rightPath) { return left.dev === right.dev && left.ino === right.ino; } +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured Markdown module could not be loaded.'); + } +} + +function runMeasuredMarkdownToHtml(markdownToHtml, source) { + try { + return markdownToHtml(source); + } catch { + throw new Error('Measured markdownToHtml() execution failed.'); + } +} + async function main() { const args = resolveArguments(process.argv.slice(2)); if ( @@ -246,12 +262,15 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const measuredModule = await import(pathToFileURL(modulePath).href); + const measuredModule = await loadMeasuredModule(modulePath); if (typeof measuredModule.markdownToHtml !== 'function') { throw new Error('Measured Markdown module must export markdownToHtml().'); } - const warmup = measuredModule.markdownToHtml(source); + const warmup = runMeasuredMarkdownToHtml( + measuredModule.markdownToHtml, + source, + ); if (typeof warmup !== 'string') { throw new Error('Measured markdownToHtml() must return a string.'); } @@ -259,7 +278,10 @@ async function main() { const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - const output = measuredModule.markdownToHtml(source); + const output = runMeasuredMarkdownToHtml( + measuredModule.markdownToHtml, + source, + ); const elapsed = performance.now() - start; if ( typeof output !== 'string' || From df31a769e8f2ce780aa30fd06e82c0c8480ec8bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:04:27 -0700 Subject: [PATCH 067/260] test(perf): require deterministic demo chunking --- src/demoBundleChunking.test.ts | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/demoBundleChunking.test.ts diff --git a/src/demoBundleChunking.test.ts b/src/demoBundleChunking.test.ts new file mode 100644 index 00000000..d6097e45 --- /dev/null +++ b/src/demoBundleChunking.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { demoVendorChunk } from '../vite.demo.chunking'; + +describe('demo bundle chunking contract', () => { + it('keeps major editor dependency families in deterministic vendor chunks', () => { + const pnpmPrefix = '/repo/node_modules/.pnpm/example/node_modules/'; + + expect(demoVendorChunk(`${pnpmPrefix}react-dom/client.js`)).toBe( + 'react-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/pm/state/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}prosemirror-state/dist/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/core/dist/index.js`)).toBe( + 'tiptap-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}marked/lib/marked.esm.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}turndown/lib/turndown.es.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}yjs/dist/yjs.mjs`)).toBe( + 'collaboration-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}lodash-es/lodash.js`)).toBe('vendor'); + }); + + it('leaves application modules to Rollup and normalizes Windows paths', () => { + expect(demoVendorChunk('/repo/demo/App.tsx')).toBeUndefined(); + expect(demoVendorChunk(String.raw`C:\repo\node_modules\react\index.js`)).toBe( + 'react-vendor', + ); + }); +}); From 0ec9731ddae2fd8729a00e6dd155885780654234 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:05:06 -0700 Subject: [PATCH 068/260] feat(perf): classify demo vendor chunks --- vite.demo.chunking.ts | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 vite.demo.chunking.ts diff --git a/vite.demo.chunking.ts b/vite.demo.chunking.ts new file mode 100644 index 00000000..87cc2834 --- /dev/null +++ b/vite.demo.chunking.ts @@ -0,0 +1,54 @@ +const NODE_MODULES = '/node_modules/'; + +function hasPackage(moduleId: string, packageName: string): boolean { + return moduleId.includes(`${NODE_MODULES}${packageName}/`); +} + +/** + * Assign large demo-only dependency families to stable Rollup vendor chunks. + * Product/library entry points are unchanged; this only keeps the standalone + * buyer demo from regressing into one oversized JavaScript payload. + */ +export function demoVendorChunk(id: string): string | undefined { + const moduleId = id.replace(/\\/g, '/'); + + if (!moduleId.includes(NODE_MODULES)) { + return undefined; + } + + if ( + hasPackage(moduleId, 'react') || + hasPackage(moduleId, 'react-dom') || + hasPackage(moduleId, 'scheduler') + ) { + return 'react-vendor'; + } + + if ( + hasPackage(moduleId, '@tiptap/pm') || + moduleId.includes(`${NODE_MODULES}prosemirror-`) + ) { + return 'prosemirror-vendor'; + } + + if (moduleId.includes(`${NODE_MODULES}@tiptap/`)) { + return 'tiptap-vendor'; + } + + if ( + hasPackage(moduleId, 'marked') || + hasPackage(moduleId, 'turndown') || + hasPackage(moduleId, 'turndown-plugin-gfm') + ) { + return 'serialization-vendor'; + } + + if ( + hasPackage(moduleId, 'yjs') || + hasPackage(moduleId, 'y-prosemirror') + ) { + return 'collaboration-vendor'; + } + + return 'vendor'; +} From 490795bfa0389841ff3c7a17ad261e86df7b0eef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:05:24 -0700 Subject: [PATCH 069/260] fix(perf): split oversized demo bundle --- vite.demo.config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vite.demo.config.ts b/vite.demo.config.ts index dd5e2d18..e5d2a0b6 100644 --- a/vite.demo.config.ts +++ b/vite.demo.config.ts @@ -1,6 +1,7 @@ import { resolve } from 'node:path'; import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; +import { demoVendorChunk } from './vite.demo.chunking'; // Standalone demo build (Vite). `pnpm build:demo` emits a static site to // dist-demo/ that can be served by any static host or the provided Dockerfile. @@ -11,5 +12,10 @@ export default defineConfig({ build: { outDir: resolve(__dirname, 'dist-demo'), emptyOutDir: true, + rollupOptions: { + output: { + manualChunks: demoVendorChunk, + }, + }, }, }); From 84aab25f818874eaddb458a5b457aa51d3ea84f7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:07:27 -0700 Subject: [PATCH 070/260] test(perf): require revision-evidence measurement harness --- ...ormanceRevisionMeasurementContract.test.ts | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 src/performanceRevisionMeasurementContract.test.ts diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts new file mode 100644 index 00000000..43c2cb21 --- /dev/null +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -0,0 +1,172 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function argumentsFor( + input: string, + modulePath: string, + output: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + fileSha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision-evidence runtime measurement contract', () => { + it('writes privacy-safe revision samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: fileSha256(modulePath), + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain( + 'Synthetic benchmark content', + ); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + expect( + JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), + ).toMatchObject({ + sampleCount: 3, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the revision API', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); + const input = join(root, 'small.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 1f64915b563a9e5db66de4b2b6299eb5506faf61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:08:09 -0700 Subject: [PATCH 071/260] feat(perf): measure packed revision-evidence latency --- benchmarks/measure-revision-evidence.mjs | 333 +++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 benchmarks/measure-revision-evidence.mjs diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs new file mode 100644 index 00000000..10040810 --- /dev/null +++ b/benchmarks/measure-revision-evidence.mjs @@ -0,0 +1,333 @@ +import { createHash } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; + +function resolveArguments(argv) { + const expectedFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Revision benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Revision benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Revision benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Revision benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Revision benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Revision benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedEnvelopeBytes(path) { + return readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Revision benchmark input must be a regular non-symlink file.', + 'Revision benchmark input exceeds the supported size.', + ); +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured revision module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + return realpathSync(resolvedPath); +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured revision module must be a local regular file.', + 'Measured revision module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Revision benchmark artifact digest does not match the measured module.', + ); + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured revision module could not be loaded.'); + } +} + +async function runMeasuredRevision(createRevisionEvidence, source) { + let evidence; + try { + evidence = await createRevisionEvidence(source); + } catch { + throw new Error('Measured revision-evidence execution failed.'); + } + if ( + typeof evidence !== 'object' || + evidence === null || + typeof evidence.revision !== 'object' || + evidence.revision === null || + typeof evidence.revision.digestHex !== 'string' || + !SHA256_PATTERN.test(evidence.revision.digestHex) + ) { + throw new Error('Measured revision-evidence result is invalid.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { + throw new Error('Revision benchmark output must not overwrite its input.'); + } + const source = readBoundedEnvelopeBytes(args.inputPath); + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Revision benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + + const measuredModule = await loadMeasuredModule(modulePath); + if ( + typeof measuredModule.createDocumentEnvelopeRevisionEvidenceBytes !== + 'function' + ) { + throw new Error( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + } + + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + const elapsed = performance.now() - start; + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Revision measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + mkdirSync(dirname(args.outputPath), { recursive: true }); + writeFileSync( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `revision-evidence-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Revision benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From 407b8a15a2db64f686b6971bf9e77daa9f8090d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:41:06 -0700 Subject: [PATCH 072/260] test(perf): reject symlinked summary inputs --- ...rmanceRegressionComparatorContract.test.ts | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 997a02ac..dfeab91e 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -202,6 +202,48 @@ describe('benchmark regression comparator contract', () => { } }); + it('rejects symlinked summary inputs instead of comparing mutable aliases', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-symlink-')); + const baselineTargetPath = join(root, 'baseline-target.json'); + const baselinePath = join(root, 'baseline-link.json'); + const currentPath = join(root, 'current.json'); + try { + writeFileSync(baselineTargetPath, `${JSON.stringify(summary())}\n`, 'utf8'); + symlinkSync(baselineTargetPath, baselinePath); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 5882974bb8d63ade28567a1823d21f918d520c90 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:44:12 -0700 Subject: [PATCH 073/260] fix(perf): reject symlinked summary evidence --- benchmarks/compare-summaries.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 975e8c97..6b2ba4be 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -2,6 +2,7 @@ import { closeSync, constants, fstatSync, + lstatSync, openSync, readSync, } from 'node:fs'; @@ -11,7 +12,9 @@ const MAX_INPUT_BYTES = 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = - constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); @@ -84,6 +87,16 @@ function resolveArguments(argv) { } function readBoundedJson(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); From 24d05fd21359813df77024f44f7a4c5df89d8820 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 11:30:30 -0700 Subject: [PATCH 074/260] test(perf): redact hostile revision result accessors --- ...ormanceRevisionMeasurementContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 43c2cb21..1db36f76 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -169,4 +169,36 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts hostile revision-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-revision-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return new Proxy({}, { get(_target, property) { if (property === 'revision') throw new Error('${privateSentinel}'); return undefined; } }); }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 10b071d106b31e9445df4c9bea2ba81d68f521fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 11:34:15 -0700 Subject: [PATCH 075/260] fix(perf): contain hostile revision result accessors --- benchmarks/measure-revision-evidence.mjs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 10040810..24b293e4 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -233,14 +233,22 @@ async function runMeasuredRevision(createRevisionEvidence, source) { } catch { throw new Error('Measured revision-evidence execution failed.'); } - if ( - typeof evidence !== 'object' || - evidence === null || - typeof evidence.revision !== 'object' || - evidence.revision === null || - typeof evidence.revision.digestHex !== 'string' || - !SHA256_PATTERN.test(evidence.revision.digestHex) - ) { + + let digestHex; + try { + if (typeof evidence !== 'object' || evidence === null) { + throw new Error('invalid revision evidence'); + } + const revision = evidence.revision; + if (typeof revision !== 'object' || revision === null) { + throw new Error('invalid revision evidence'); + } + digestHex = revision.digestHex; + } catch { + throw new Error('Measured revision-evidence result is invalid.'); + } + + if (typeof digestHex !== 'string' || !SHA256_PATTERN.test(digestHex)) { throw new Error('Measured revision-evidence result is invalid.'); } } From 286bc0762e068797d2ecf6362b733df7dd3b13a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:01:53 -0700 Subject: [PATCH 076/260] test(perf): define retained-memory settling RED contract --- src/performanceMemorySettlingContract.test.ts | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 src/performanceMemorySettlingContract.test.ts diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts new file mode 100644 index 00000000..c8602354 --- /dev/null +++ b/src/performanceMemorySettlingContract.test.ts @@ -0,0 +1,133 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/analyze-memory-settling.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type MemoryEvidenceOverrides = Partial<{ + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + warmupSamples: number; + samples: number[]; +}>; + +function evidence(overrides: MemoryEvidenceOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + warmupSamples: 2, + samples: [900, 920, 1000, 1005, 995, 1010, 1015, 1008, 1012, 1010], + ...overrides, + }; +} + +function runAnalysis( + root: string, + input: ReturnType, + maxGrowthBytes: string, +) { + const inputPath = join(root, 'memory-evidence.json'); + writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + maxGrowthBytes, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('retained-memory settling evidence contract', () => { + it('passes bounded settled growth using explicit warmup and comparison windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-pass-')); + try { + const result = runAnalysis(root, evidence(), '16'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 10, + warmupSamples: 2, + windowSize: 3, + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1010, + retainedGrowthBytes: 10, + maxGrowthBytes: 16, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a retained-memory growth breach while preserving the public receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-fail-')); + try { + const result = runAnalysis( + root, + evidence({ + samples: [900, 920, 1000, 1005, 995, 1100, 1120, 1110, 1130, 1140], + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toMatchObject({ + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1130, + retainedGrowthBytes: 130, + maxGrowthBytes: 50, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence that cannot supply two disjoint settled windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-short-')); + try { + const result = runAnalysis( + root, + evidence({ warmupSamples: 2, samples: [900, 920, 1000, 1005, 1010] }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From ef33eba152daf3144acd9101322e71df98c68c70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:05:21 -0700 Subject: [PATCH 077/260] feat(perf): analyze retained-memory settling --- benchmarks/analyze-memory-settling.mjs | 283 +++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 benchmarks/analyze-memory-settling.mjs diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs new file mode 100644 index 00000000..9f5009d3 --- /dev/null +++ b/benchmarks/analyze-memory-settling.mjs @@ -0,0 +1,283 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const EVIDENCE_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'warmupSamples', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 6 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--window-size' || + argv[3].trim().length === 0 || + argv[4] !== '--max-growth-bytes' || + argv[5].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/analyze-memory-settling.mjs --input --window-size --max-growth-bytes ', + ); + } + + const windowSize = Number(argv[3]); + if (!Number.isSafeInteger(windowSize) || windowSize <= 0) { + throw new Error('Memory settling window size must be a positive safe integer.'); + } + + const maxGrowthBytes = Number(argv[5]); + if (!Number.isFinite(maxGrowthBytes) || maxGrowthBytes < 0) { + throw new Error( + 'Memory settling max growth bytes must be a finite non-negative number.', + ); + } + + return Object.freeze({ + inputPath: resolve(argv[1]), + windowSize, + maxGrowthBytes, + }); +} + +function readBoundedJson(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Memory settling evidence input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Memory settling evidence input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error( + 'Memory settling evidence input exceeds the supported size.', + ); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Memory settling evidence input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Memory settling evidence input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateEvidence(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Memory settling evidence input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== EVIDENCE_KEYS.size || + keys.some((key) => !EVIDENCE_KEYS.has(key)) + ) { + throw new Error('Memory settling evidence input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Memory settling evidence contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Memory settling evidence benchmarkId is invalid.'); + } + if (value.unit !== 'bytes') { + throw new Error('Memory settling evidence unit must be bytes.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Memory settling evidence sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Memory settling evidence artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Memory settling evidence documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Memory settling evidence runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Memory settling evidence referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.warmupSamples) || + value.warmupSamples < 0 || + value.warmupSamples > MAX_SAMPLES + ) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES || + value.samples.some( + (sample) => + !Number.isSafeInteger(sample) || sample < 0, + ) + ) { + throw new Error( + 'Memory settling evidence samples must be bounded non-negative safe integers.', + ); + } + if (value.warmupSamples >= value.samples.length) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + warmupSamples: value.warmupSamples, + samples: Object.freeze([...value.samples]), + }); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[midpoint]; + return (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +function analyze(evidence, windowSize, maxGrowthBytes) { + const settledSamples = evidence.samples.slice(evidence.warmupSamples); + if (settledSamples.length < windowSize * 2) { + throw new Error( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } + + const firstWindowMedianBytes = median(settledSamples.slice(0, windowSize)); + const lastWindowMedianBytes = median(settledSamples.slice(-windowSize)); + const retainedGrowthBytes = lastWindowMedianBytes - firstWindowMedianBytes; + + return Object.freeze({ + contractVersion: 1, + benchmarkId: evidence.benchmarkId, + unit: evidence.unit, + sourceCommitSha: evidence.sourceCommitSha, + artifactSha256: evidence.artifactSha256, + documentProfile: evidence.documentProfile, + runtimeId: evidence.runtimeId, + referenceHardwareId: evidence.referenceHardwareId, + sampleCount: evidence.samples.length, + warmupSamples: evidence.warmupSamples, + windowSize, + firstWindowMedianBytes, + lastWindowMedianBytes, + retainedGrowthBytes, + maxGrowthBytes, + passed: retainedGrowthBytes <= maxGrowthBytes, + }); +} + +function main() { + const { inputPath, windowSize, maxGrowthBytes } = resolveArguments( + process.argv.slice(2), + ); + const evidence = validateEvidence(readBoundedJson(inputPath)); + const result = analyze(evidence, windowSize, maxGrowthBytes); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Memory settling analysis failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From e4cab92ef0693fe37c5e2d0208750c6dd96cbcc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:19:34 -0700 Subject: [PATCH 078/260] test(perf): reject mismatched memory evidence profiles --- src/performanceMemorySettlingContract.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index c8602354..ef44d607 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -9,6 +9,7 @@ const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); type MemoryEvidenceOverrides = Partial<{ + benchmarkId: string; sourceCommitSha: string; artifactSha256: string; documentProfile: string; @@ -130,4 +131,26 @@ describe('retained-memory settling evidence contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); + try { + const result = runAnalysis( + root, + evidence({ + benchmarkId: 'editor-lifecycle-retained-memory-small', + documentProfile: 'large', + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e671a5a9c40742e2a699016bf15277a590aad5ac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 15:36:47 -0700 Subject: [PATCH 079/260] fix(perf): bind memory benchmark id to profile --- benchmarks/analyze-memory-settling.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 9f5009d3..50a9f095 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -15,6 +15,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PREFIX = 'editor-lifecycle-retained-memory-'; const BENCHMARK_ID_PATTERN = /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -175,6 +176,13 @@ function validateEvidence(value) { ) { throw new Error('Memory settling evidence documentProfile is invalid.'); } + if ( + value.benchmarkId.slice(BENCHMARK_ID_PREFIX.length) !== value.documentProfile + ) { + throw new Error( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From 5a6944c096665b96c307852524e8bbe58db16369 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 17:03:01 -0700 Subject: [PATCH 080/260] test(perf): reject inexact memory medians --- src/performanceMemorySettlingContract.test.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index ef44d607..93819962 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -39,6 +39,7 @@ function runAnalysis( root: string, input: ReturnType, maxGrowthBytes: string, + windowSize = '3', ) { const inputPath = join(root, 'memory-evidence.json'); writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); @@ -49,7 +50,7 @@ function runAnalysis( '--input', inputPath, '--window-size', - '3', + windowSize, '--max-growth-bytes', maxGrowthBytes, ], @@ -153,4 +154,28 @@ describe('retained-memory settling evidence contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before a precision-loss false green from an inexact even-window median', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-precision-')); + try { + const maxSafe = Number.MAX_SAFE_INTEGER; + const result = runAnalysis( + root, + evidence({ + warmupSamples: 0, + samples: [maxSafe - 1, maxSafe - 1, maxSafe - 1, maxSafe], + }), + '0.25', + '2', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling window median must be exactly representable.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From a1b5201b1d1708a8c61989987f0eca0b27e9db10 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 17:07:21 -0700 Subject: [PATCH 081/260] fix(perf): fail closed on inexact memory medians --- benchmarks/analyze-memory-settling.mjs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 50a9f095..9e60c947 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -236,7 +236,17 @@ function median(values) { const sorted = [...values].sort((left, right) => left - right); const midpoint = Math.floor(sorted.length / 2); if (sorted.length % 2 === 1) return sorted[midpoint]; - return (sorted[midpoint - 1] + sorted[midpoint]) / 2; + + const left = sorted[midpoint - 1]; + const right = sorted[midpoint]; + const distance = right - left; + const wholeMidpoint = left + Math.floor(distance / 2); + if (distance % 2 === 1 && wholeMidpoint >= 2 ** 52) { + throw new Error( + 'Memory settling window median must be exactly representable.', + ); + } + return wholeMidpoint + (distance % 2) / 2; } function analyze(evidence, windowSize, maxGrowthBytes) { From 80c8bbb95a0072d87b30a783bd6b9dc83d5b2767 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:29:25 -0700 Subject: [PATCH 082/260] test(perf): reject mismatched benchmark profiles --- ...gressionProfileConsistencyContract.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/performanceRegressionProfileConsistencyContract.test.ts diff --git a/src/performanceRegressionProfileConsistencyContract.test.ts b/src/performanceRegressionProfileConsistencyContract.test.ts new file mode 100644 index 00000000..bf787527 --- /dev/null +++ b/src/performanceRegressionProfileConsistencyContract.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression profile consistency contract', () => { + it('rejects summaries whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-profile-')); + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + + try { + writeFileSync(baselinePath, `${JSON.stringify(summary('b'.repeat(64)))}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(summary('c'.repeat(64)))}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From dcf8b65b3a97979c8b9cf176e18b699247648c00 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:33:14 -0700 Subject: [PATCH 083/260] fix(perf): bind benchmark ids to document profiles --- benchmarks/compare-summaries.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 6b2ba4be..4b891f14 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -187,6 +187,9 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary documentProfile is invalid.'); } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark summary profile must match documentProfile.'); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From b240eace91de6f2c6a91fdc752ec0cd96c1bf1b8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:35:29 -0700 Subject: [PATCH 084/260] test(perf): reject mismatched sample profiles --- ...eSummaryProfileConsistencyContract.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/performanceSummaryProfileConsistencyContract.test.ts diff --git a/src/performanceSummaryProfileConsistencyContract.test.ts b/src/performanceSummaryProfileConsistencyContract.test.ts new file mode 100644 index 00000000..118346de --- /dev/null +++ b/src/performanceSummaryProfileConsistencyContract.test.ts @@ -0,0 +1,47 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary profile consistency contract', () => { + it('rejects sample evidence whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-profile-')); + const inputPath = join(root, 'samples.json'); + const outputDirectory = join(root, 'summary'); + + try { + writeFileSync( + inputPath, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [70, 80, 90, 100], + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 00b062e680ba6d7395a1a3d55cd923f9b097b2ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:38:21 -0700 Subject: [PATCH 085/260] fix(perf): bind sample ids to document profiles --- benchmarks/summarize-samples.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 354c2539..58d00536 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -151,6 +151,9 @@ function validateInput(value) { ) { throw new Error('Benchmark documentProfile is invalid.'); } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark sample profile must match documentProfile.'); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From 1a76294f71eb4f8f17505b3bc773879c54373bf5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:05:51 -0700 Subject: [PATCH 086/260] test(perf): reject corpus output symlink escape --- src/performanceCorpusContract.test.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index e92c7f2c..02bc5c02 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -85,4 +92,20 @@ describe('deterministic synthetic performance corpus', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of following a corpus output symlink', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.md'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'small.md')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From abf7d8944b83fc2e538b19b939450a9e428d2ddd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:10:50 -0700 Subject: [PATCH 087/260] fix(perf): protect corpus output paths --- benchmarks/generate-corpus.mjs | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index af73290e..af9d85e9 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -1,5 +1,11 @@ import { createHash } from 'node:crypto'; -import { mkdirSync, writeFileSync } from 'node:fs'; +import { + lstatSync, + mkdirSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const RASTER_FIXTURES = Object.freeze([ @@ -42,6 +48,24 @@ const SCRIPT_LABELS = Object.freeze([ 'mixed', ]); +let outputWriteCounter = 0; + +function writeRegularOutput(path, content) { + const existing = lstatSync(path, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isFile()) { + throw new Error('Benchmark corpus output must be a regular file.'); + } + + const temporaryPath = `${path}.tmp-${process.pid}-${outputWriteCounter}`; + outputWriteCounter += 1; + try { + writeFileSync(temporaryPath, content, { flag: 'wx' }); + renameSync(temporaryPath, path); + } finally { + rmSync(temporaryPath, { force: true }); + } +} + function buildSection(index) { const id = String(index).padStart(4, '0'); const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { @@ -111,7 +135,7 @@ const profileManifest = {}; for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { const body = buildProfile(profile, sections); const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, `${profile}.md`), bytes); + writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); profileManifest[profile] = Object.freeze({ sections, bytes: bytes.byteLength, @@ -125,8 +149,7 @@ const manifest = Object.freeze({ scripts: SCRIPT_LABELS, profiles: profileManifest, }); -writeFileSync( +writeRegularOutput( resolve(outputDirectory, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', ); From 4002bc46202de7757f2274997d60bdb24c89e259 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:54:27 -0700 Subject: [PATCH 088/260] test(perf): redact benchmark output path failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index 4f6a08c3..9b81bdec 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -19,11 +19,14 @@ function sha256(value: string) { return createHash('sha256').update(value, 'utf8').digest('hex'); } -function runMeasurement(moduleSource: string) { +function runMeasurement( + moduleSource: string, + outputForRoot: (root: string) => string = (root) => join(root, 'samples.json'), +) { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); const input = join(root, 'document.md'); const modulePath = join(root, 'measured.mjs'); - const output = join(root, 'samples.json'); + const output = outputForRoot(root); writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); writeFileSync(modulePath, moduleSource, 'utf8'); @@ -93,4 +96,26 @@ describe('Markdown measurement error privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output path traversal fails', () => { + const privateSentinel = 'private-output-sentinel-must-not-leak'; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const { root, output, result } = runMeasurement(moduleSource, (testRoot) => { + const blockedParent = join(testRoot, privateSentinel); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + return join(blockedParent, 'samples.json'); + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 1528f0eb3a53aa86453e8e77f74adcd6d743c8c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:57:01 -0700 Subject: [PATCH 089/260] fix(perf): redact benchmark output path failures --- benchmarks/measure-markdown.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index c50733c7..cbe42a96 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -215,8 +215,16 @@ function verifyMeasuredModuleDigest(modulePath, expectedSha256) { } } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output path could not be inspected.'); + } +} + function refersToSameFile(leftPath, rightPath) { - const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; if (!rightMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); @@ -294,7 +302,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } From da69be66ec10b4345ad95209df3367f37570ac17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:01:19 -0700 Subject: [PATCH 090/260] test(perf): redact revision output path failures --- ...ormanceRevisionMeasurementContract.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 1db36f76..8c604a8c 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -201,4 +201,38 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-output-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = 'private-revision-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const samplesPath = join(blockedParent, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 523c98605e3a24ab94154ed8d56c238cd2a100a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:04:46 -0700 Subject: [PATCH 091/260] fix(perf): redact revision output path failures --- benchmarks/measure-revision-evidence.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 24b293e4..d830cb5d 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -207,8 +207,16 @@ function verifyMeasuredModuleDigest(modulePath, expectedSha256) { } } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output path could not be inspected.'); + } +} + function refersToSameFile(leftPath, rightPath) { - const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; if (!rightMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); @@ -303,7 +311,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } From 28a45ac43b48c92d4124b6f71d699ca8875bcfc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:06:45 -0700 Subject: [PATCH 092/260] test(perf): redact summary output preparation failures --- ...rformanceMeasurementOutputContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index 45376b1f..895732ad 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -164,4 +164,36 @@ describe('benchmark summary output contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when the output directory cannot be prepared', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-privacy-')); + const input = join(root, 'samples.json'); + const privateSentinel = 'private-summary-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const output = join(blockedParent, 'output'); + try { + writeValidInput(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory could not be prepared.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From d44925ea6ca13a6f5f5233bc70e56018a1bc6273 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 22:03:11 -0700 Subject: [PATCH 093/260] fix(perf): redact output directory preparation failures --- benchmarks/summarize-samples.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 58d00536..65592796 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -253,11 +253,19 @@ function assertRegularOutputDestination(path) { } } +function prepareOutputDirectory(path) { + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); const summaryTextPath = resolve(outputDirectory, 'summary.txt'); - mkdirSync(outputDirectory, { recursive: true }); + prepareOutputDirectory(outputDirectory); if ( inputPath === summaryJsonPath || inputPath === summaryTextPath || From d4541760f8036ce1c637a87529eb9451fd673a9d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:01:44 -0700 Subject: [PATCH 094/260] test(perf): redact Markdown output publication failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index 9b81bdec..c069b76f 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -118,4 +118,26 @@ describe('Markdown measurement error privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const privateSentinel = `private-markdown-publication-${process.pid}`; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const blockedOutput = join('/sys', privateSentinel, 'samples.json'); + const { root, output, result } = runMeasurement( + moduleSource, + () => blockedOutput, + ); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e066321ef7fd40f256dbbd7d75db04968459d62e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:05:39 -0700 Subject: [PATCH 095/260] fix(perf): redact Markdown output publication failures --- benchmarks/measure-markdown.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index cbe42a96..e240af3c 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -250,6 +250,15 @@ function runMeasuredMarkdownToHtml(markdownToHtml, source) { } } +function writeMeasurementOutput(path, content) { + try { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, 'utf8'); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } +} + async function main() { const args = resolveArguments(process.argv.slice(2)); if ( @@ -306,8 +315,7 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } - mkdirSync(dirname(args.outputPath), { recursive: true }); - writeFileSync( + writeMeasurementOutput( args.outputPath, `${JSON.stringify( { @@ -324,7 +332,6 @@ async function main() { null, 2, )}\n`, - 'utf8', ); } From 0b724e0263403adb18affe6ca16cb1914991a929 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:14:23 -0700 Subject: [PATCH 096/260] test(perf): redact revision output publication failures --- ...ormanceRevisionMeasurementContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 8c604a8c..d7258f6a 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -235,4 +235,36 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-publication-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = `private-revision-publication-${process.pid}`; + const samplesPath = join('/sys', privateSentinel, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 12a48b402a680eb5b4d5722e5185db6270f42ab6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:33:45 -0700 Subject: [PATCH 097/260] fix(perf): redact revision output publication failures --- benchmarks/measure-revision-evidence.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index d830cb5d..2bec7367 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -234,6 +234,15 @@ async function loadMeasuredModule(modulePath) { } } +function writeMeasurementOutput(path, content) { + try { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, 'utf8'); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } +} + async function runMeasuredRevision(createRevisionEvidence, source) { let evidence; try { @@ -315,8 +324,7 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } - mkdirSync(dirname(args.outputPath), { recursive: true }); - writeFileSync( + writeMeasurementOutput( args.outputPath, `${JSON.stringify( { @@ -333,7 +341,6 @@ async function main() { null, 2, )}\n`, - 'utf8', ); } From 86a771e0ce8994b195616fe67a78da83dbdbd2c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 04:32:57 -0700 Subject: [PATCH 098/260] test(perf): require single-command benchmark suite --- ...formanceSingleCommandSuiteContract.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/performanceSingleCommandSuiteContract.test.ts diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts new file mode 100644 index 00000000..4747d273 --- /dev/null +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -0,0 +1,95 @@ +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('single-command benchmark suite contract', () => { + it('measures and summarizes one deterministic Markdown profile with one command', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const inputPath = join(directory, 'input.md'); + const modulePath = join(directory, 'measured.mjs'); + const outputDirectory = join(directory, 'evidence'); + const moduleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const artifactSha256 = createHash('sha256') + .update(moduleSource) + .digest('hex'); + + const output = execFileSync( + process.execPath, + [ + suitePath, + '--input', + inputPath, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(JSON.parse(output.trim())).toEqual({ + contractVersion: 1, + documentProfile: 'small', + samples: 'samples.json', + status: 'completed', + summaryJson: 'summary/summary.json', + summaryText: 'summary/summary.txt', + }); + + const samples = JSON.parse( + readFileSync(join(outputDirectory, 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(samples.benchmarkId).toBe('markdown-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync(join(outputDirectory, 'summary', 'summary.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(summary.benchmarkId).toBe('markdown-serialization-small'); + expect(summary.documentProfile).toBe('small'); + expect( + readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), + ).toContain('markdown-serialization-small'); + }); +}); From 9a618bcf05f4f49d4f2fe62d833fbf7e0ad3a84b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 04:36:38 -0700 Subject: [PATCH 099/260] feat(perf): compose single-command benchmark suite --- benchmarks/run-current-suite.mjs | 94 ++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 benchmarks/run-current-suite.mjs diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs new file mode 100644 index 00000000..95354bdf --- /dev/null +++ b/benchmarks/run-current-suite.mjs @@ -0,0 +1,94 @@ +import { spawnSync } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const expectedFlags = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function resolveArguments(argv) { + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + return Object.freeze({ + documentProfile: argv[5], + forwardedArguments: Object.freeze(argv.slice(0, -2)), + outputDirectory: resolve(argv[17]), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function main(argv) { + const args = resolveArguments(argv); + const samplesPath = resolve(args.outputDirectory, 'samples.json'); + const summaryDirectory = resolve(args.outputDirectory, 'summary'); + + runBoundedNodeScript( + 'measure-markdown.mjs', + [...args.forwardedArguments, '--output', samplesPath], + 'Benchmark suite measurement failed.', + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + contractVersion: 1, + documentProfile: args.documentProfile, + samples: 'samples.json', + status: 'completed', + summaryJson: 'summary/summary.json', + summaryText: 'summary/summary.txt', + })}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From c41f1017bc64aa79f730ef35e8f33ea0dc6684a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:07:11 -0700 Subject: [PATCH 100/260] test(perf): reject symlink benchmark output directories --- ...formanceSingleCommandSuiteContract.test.ts | 118 +++++++++++++----- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 4747d273..611fb419 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -1,9 +1,12 @@ import { createHash } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { + existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -21,44 +24,64 @@ afterEach(() => { } }); +function benchmarkArguments( + inputPath: string, + modulePath: string, + artifactSha256: string, + outputDirectory: string, +): string[] { + return [ + suitePath, + '--input', + inputPath, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function writeBenchmarkInputs(directory: string): { + artifactSha256: string; + inputPath: string; + modulePath: string; +} { + const inputPath = join(directory, 'input.md'); + const modulePath = join(directory, 'measured.mjs'); + const moduleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + return { + artifactSha256: createHash('sha256').update(moduleSource).digest('hex'), + inputPath, + modulePath, + }; +} + describe('single-command benchmark suite contract', () => { it('measures and summarizes one deterministic Markdown profile with one command', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); - const inputPath = join(directory, 'input.md'); - const modulePath = join(directory, 'measured.mjs'); const outputDirectory = join(directory, 'evidence'); - const moduleSource = - "export function markdownToHtml(source) { return `

${source}

`; }\n"; - writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); - writeFileSync(modulePath, moduleSource, 'utf8'); - const artifactSha256 = createHash('sha256') - .update(moduleSource) - .digest('hex'); + const { artifactSha256, inputPath, modulePath } = + writeBenchmarkInputs(directory); const output = execFileSync( process.execPath, - [ - suitePath, - '--input', - inputPath, - '--module', - modulePath, - '--profile', - 'small', - '--samples', - '2', - '--source-commit-sha', - 'a'.repeat(40), - '--artifact-sha256', - artifactSha256, - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), { cwd: repositoryRoot, encoding: 'utf8', @@ -92,4 +115,37 @@ describe('single-command benchmark suite contract', () => { readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), ).toContain('markdown-serialization-small'); }); + + it('fails closed before writing evidence through a symlink output directory', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const { artifactSha256, inputPath, modulePath } = + writeBenchmarkInputs(directory); + const actualOutputDirectory = join(directory, 'outside-target'); + const outputDirectory = join(directory, 'evidence-link'); + mkdirSync(actualOutputDirectory); + symlinkSync( + actualOutputDirectory, + outputDirectory, + process.platform === 'win32' ? 'junction' : 'dir', + ); + + const result = spawnSync( + process.execPath, + benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toBe( + 'Benchmark suite output directory must be a non-symlink directory.\n', + ); + expect(existsSync(join(actualOutputDirectory, 'samples.json'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'summary'))).toBe(false); + }); }); From 18fe7381a94a8c03b2897ae38198435617f8236e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:11:20 -0700 Subject: [PATCH 101/260] fix(perf): reject symlink suite output roots --- benchmarks/run-current-suite.mjs | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 95354bdf..304cc2bc 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,4 +1,5 @@ import { spawnSync } from 'node:child_process'; +import { lstatSync, mkdirSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -15,6 +16,8 @@ const expectedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; function resolveArguments(argv) { if ( @@ -34,6 +37,39 @@ function resolveArguments(argv) { }); } +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function prepareOutputDirectory(path) { + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (existing.isSymbolicLink() || !existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + const created = inspectOutputDirectory(path); + if ( + created === undefined || + created.isSymbolicLink() || + !created.isDirectory() + ) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } +} + function runBoundedNodeScript(scriptName, args, failureMessage) { const result = spawnSync( process.execPath, @@ -58,6 +94,7 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { function main(argv) { const args = resolveArguments(argv); + prepareOutputDirectory(args.outputDirectory); const samplesPath = resolve(args.outputDirectory, 'samples.json'); const summaryDirectory = resolve(args.outputDirectory, 'summary'); From 156b8e3463e1cef190e139cc39dda63c0e31997c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:22:14 -0700 Subject: [PATCH 102/260] test(perf): reject symlink summary output directory --- ...manceMeasurementStatisticsContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 3c477543..7f7627b3 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, rmSync, + symlinkSync, truncateSync, writeFileSync, } from 'node:fs'; @@ -290,4 +291,35 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before writing summaries through a symlink output directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-symlink-')); + const input = join(root, 'samples.json'); + const target = join(root, 'outside-target'); + const output = join(root, 'output-link'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(target); + symlinkSync(target, output, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(target, 'summary.json'))).toBe(false); + expect(existsSync(join(target, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 52ba7337829822b75569c6e8cc718c92764738a9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:56:38 -0700 Subject: [PATCH 103/260] fix(perf): reject symlink summary directories --- benchmarks/summarize-samples.mjs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 65592796..f93be873 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -254,11 +254,32 @@ function assertRegularOutputDestination(path) { } function prepareOutputDirectory(path) { + const current = lstatSync(path, { throwIfNoEntry: false }); + if (current !== undefined) { + if (current.isSymbolicLink() || !current.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + return; + } + try { mkdirSync(path, { recursive: true }); } catch { throw new Error('Benchmark summary output directory could not be prepared.'); } + + const created = lstatSync(path, { throwIfNoEntry: false }); + if ( + created === undefined || + created.isSymbolicLink() || + !created.isDirectory() + ) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } } function main() { From bd3eb04ff6c9cc7a1c2d0943a9cd643656aefcd4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:58:54 -0700 Subject: [PATCH 104/260] test(perf): reject symlinked summary ancestors --- ...asurementStatisticsAncestorSymlink.test.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/performanceMeasurementStatisticsAncestorSymlink.test.ts diff --git a/src/performanceMeasurementStatisticsAncestorSymlink.test.ts b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts new file mode 100644 index 00000000..aae380be --- /dev/null +++ b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output path ancestry', () => { + it('fails closed before writing through a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-ancestor-')); + const input = join(root, 'samples.json'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + writeInput(input); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'summary.json'))).toBe(false); + expect(existsSync(join(outside, 'nested-output', 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 3b722e44f248a406160fd09dbb3f41d3f8f094b7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:59:41 -0700 Subject: [PATCH 105/260] fix(perf): reject symlinked summary ancestors --- benchmarks/summarize-samples.mjs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index f93be873..da952e53 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -10,7 +10,7 @@ import { statSync, writeFileSync, } from 'node:fs'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -253,10 +253,26 @@ function assertRegularOutputDestination(path) { } } +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = lstatSync(current, { throwIfNoEntry: false }); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); const current = lstatSync(path, { throwIfNoEntry: false }); if (current !== undefined) { - if (current.isSymbolicLink() || !current.isDirectory()) { + if (!current.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', ); @@ -270,12 +286,9 @@ function prepareOutputDirectory(path) { throw new Error('Benchmark summary output directory could not be prepared.'); } + assertNoSymlinkDirectoryComponents(path); const created = lstatSync(path, { throwIfNoEntry: false }); - if ( - created === undefined || - created.isSymbolicLink() || - !created.isDirectory() - ) { + if (created === undefined || !created.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', ); From eb15b2f1a35fbd9a9eb61ad8332b19024e531980 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:00:43 -0700 Subject: [PATCH 106/260] test(perf): reject symlinked suite ancestors --- ...nceMeasurementSuiteAncestorSymlink.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/performanceMeasurementSuiteAncestorSymlink.test.ts diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts new file mode 100644 index 00000000..7b61c4b5 --- /dev/null +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/run-current-suite.mjs'); + +describe('benchmark suite output path ancestry', () => { + it('fails closed before preparing an output beneath a symlinked ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-ancestor-')); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + join(root, 'unused.md'), + '--module', + join(root, 'unused.mjs'), + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 0cfd5a9e86ab429e4ef3b6a078af7d3fa09c5a13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:01:11 -0700 Subject: [PATCH 107/260] fix(perf): reject symlinked suite ancestors --- benchmarks/run-current-suite.mjs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 304cc2bc..5d9471b2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -45,10 +45,24 @@ function inspectOutputDirectory(path) { } } +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); const existing = inspectOutputDirectory(path); if (existing !== undefined) { - if (existing.isSymbolicLink() || !existing.isDirectory()) { + if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } return; @@ -60,12 +74,9 @@ function prepareOutputDirectory(path) { throw new Error('Benchmark suite output directory could not be prepared.'); } + assertNoSymlinkDirectoryComponents(path); const created = inspectOutputDirectory(path); - if ( - created === undefined || - created.isSymbolicLink() || - !created.isDirectory() - ) { + if (created === undefined || !created.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } } From f1adf747086fa4c305f82af349599257e6c2d174 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:04:04 -0700 Subject: [PATCH 108/260] test(perf): reject symlinked markdown output ancestors --- ...wnMeasurementOutputAncestorSymlink.test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts diff --git a/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..cc60c3cb --- /dev/null +++ b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,76 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('Markdown benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-output-ancestor-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source.length}

`; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 0ac8712eea1366c6582b2f6b1da7e963295db935 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:01 -0700 Subject: [PATCH 109/260] fix(perf): reject symlinked markdown output ancestors --- benchmarks/measure-markdown.mjs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e240af3c..eb1b19b7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -28,6 +28,8 @@ const RUNTIME_ID_PATTERN = /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Markdown benchmark output directory must be a non-symlink directory.'; function resolveArguments(argv) { const expectedFlags = [ @@ -223,6 +225,27 @@ function inspectOutputPath(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; @@ -251,8 +274,14 @@ function runMeasuredMarkdownToHtml(markdownToHtml, source) { } function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); try { mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { writeFileSync(path, content, 'utf8'); } catch { throw new Error('Markdown benchmark output could not be written.'); @@ -261,6 +290,7 @@ function writeMeasurementOutput(path, content) { async function main() { const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) @@ -311,6 +341,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); From 2bcf06c9872aa497f6d1692928c64a790749de27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:20 -0700 Subject: [PATCH 110/260] test(perf): reject symlinked revision output ancestors --- ...onMeasurementOutputAncestorSymlink.test.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts diff --git a/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..48348e12 --- /dev/null +++ b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,84 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-revision-evidence.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('revision benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-output-ancestor-')); + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync( + input, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { type: 'doc', content: [] }, + }), + 'utf8', + ); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 2e9332e8bb350c9cfb2596a131df6b4878e350eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:56 -0700 Subject: [PATCH 111/260] fix(perf): reject symlinked revision output ancestors --- benchmarks/measure-revision-evidence.mjs | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 2bec7367..3c413590 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -27,6 +27,8 @@ const RUNTIME_ID_PATTERN = /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Revision benchmark output directory must be a non-symlink directory.'; function resolveArguments(argv) { const expectedFlags = [ @@ -215,6 +217,27 @@ function inspectOutputPath(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; @@ -235,8 +258,14 @@ async function loadMeasuredModule(modulePath) { } function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); try { mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { writeFileSync(path, content, 'utf8'); } catch { throw new Error('Revision benchmark output could not be written.'); @@ -272,6 +301,7 @@ async function runMeasuredRevision(createRevisionEvidence, source) { async function main() { const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) @@ -320,6 +350,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); From ea00fe920a82601ab01cabca43c8acf00354b2eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:09:44 -0700 Subject: [PATCH 112/260] fix(perf): preserve path-redacted summary failures --- benchmarks/summarize-samples.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index da952e53..369a5c45 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -253,10 +253,18 @@ function assertRegularOutputDestination(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + function assertNoSymlinkDirectoryComponents(path) { let current = path; while (true) { - const metadata = lstatSync(current, { throwIfNoEntry: false }); + const metadata = inspectOutputDirectoryComponent(current); if (metadata?.isSymbolicLink()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', @@ -270,7 +278,7 @@ function assertNoSymlinkDirectoryComponents(path) { function prepareOutputDirectory(path) { assertNoSymlinkDirectoryComponents(path); - const current = lstatSync(path, { throwIfNoEntry: false }); + const current = inspectOutputDirectoryComponent(path); if (current !== undefined) { if (!current.isDirectory()) { throw new Error( @@ -287,7 +295,7 @@ function prepareOutputDirectory(path) { } assertNoSymlinkDirectoryComponents(path); - const created = lstatSync(path, { throwIfNoEntry: false }); + const created = inspectOutputDirectoryComponent(path); if (created === undefined || !created.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', From dba75050566ec2867afa15328352f0e68e057588 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:17:35 -0700 Subject: [PATCH 113/260] test(perf): reject symlinked sample input --- ...eMeasurementStatisticsInputSymlink.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/performanceMeasurementStatisticsInputSymlink.test.ts diff --git a/src/performanceMeasurementStatisticsInputSymlink.test.ts b/src/performanceMeasurementStatisticsInputSymlink.test.ts new file mode 100644 index 00000000..0c481d53 --- /dev/null +++ b/src/performanceMeasurementStatisticsInputSymlink.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function validSamples(): string { + return `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`; +} + +describe('benchmark summary sample input file authority', () => { + it('fails closed instead of following a symlinked sample input', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-symlink-')); + const target = join(root, 'private-samples.json'); + const alias = join(root, 'samples.json'); + const output = join(root, 'summary'); + try { + writeFileSync(target, validSamples(), 'utf8'); + symlinkSync(target, alias, process.platform === 'win32' ? 'file' : undefined); + + const result = spawnSync( + process.execPath, + [script, '--input', alias, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 9afff917c557173af9f3defccde37b0f8c284b48 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:34:25 -0700 Subject: [PATCH 114/260] fix(perf): reject symlinked summary sample input --- benchmarks/summarize-samples.mjs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 369a5c45..2a99eba8 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -15,8 +15,10 @@ import { dirname, resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; -const READ_ONLY_NONBLOCKING = - constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); +const READ_ONLY_NONBLOCKING_NOFOLLOW = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); @@ -59,11 +61,24 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error('Benchmark sample input must be a regular file.'); + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); } if (metadata.size > MAX_INPUT_BYTES) { throw new Error('Benchmark sample input exceeds the supported size.'); From 824561c84a398cd6c396f2c1cdb52f3b7823e5cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:39:54 -0700 Subject: [PATCH 115/260] fix(perf): preserve nonregular input diagnostics --- benchmarks/summarize-samples.mjs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 2a99eba8..c2ff872e 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -62,23 +62,20 @@ function resolveArguments(argv) { function readBoundedJson(path) { const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - if ( - pathMetadata === undefined || - pathMetadata.isSymbolicLink() || - !pathMetadata.isFile() - ) { + if (pathMetadata?.isSymbolicLink()) { throw new Error( 'Benchmark sample input must be a regular non-symlink file.', ); } + if (pathMetadata === undefined || !pathMetadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error( - 'Benchmark sample input must be a regular non-symlink file.', - ); + throw new Error('Benchmark sample input must be a regular file.'); } if (metadata.size > MAX_INPUT_BYTES) { throw new Error('Benchmark sample input exceeds the supported size.'); From a7b6bc9dcea44142395832c8a1bb541f9b89aac2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:08:16 -0700 Subject: [PATCH 116/260] test(perf): cover markdown input path privacy --- ...ownMeasurementInputPrivacyContract.test.ts | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/performanceMarkdownMeasurementInputPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts new file mode 100644 index 00000000..748e94a1 --- /dev/null +++ b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +describe('Markdown measurement input error privacy contract', () => { + it('redacts private filesystem details when input traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-input-privacy-')); + const privateSentinel = 'private-input-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From dd35f070ba5d8852d5eb9de081d071d0ccb69bbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:13:07 -0700 Subject: [PATCH 117/260] fix(perf): redact markdown input path errors --- benchmarks/measure-markdown.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index eb1b19b7..815630f7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -103,7 +103,12 @@ function resolveArguments(argv) { } function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } if ( pathMetadata === undefined || pathMetadata.isSymbolicLink() || @@ -112,7 +117,12 @@ function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversize throw new Error(invalidFileMessage); } - const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From 1781400353a49d20634458093ce1dc3e24735973 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:19:09 -0700 Subject: [PATCH 118/260] test(perf): cover markdown module path privacy --- ...asurementModulePathPrivacyContract.test.ts | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..352e7f4d --- /dev/null +++ b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,65 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +describe('Markdown measurement module path error privacy contract', () => { + it('redacts private filesystem details when module path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-privacy-')); + const privateSentinel = 'private-module-sentinel-must-not-leak'; + const input = join(root, 'document.md'); + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'measured.mjs'); + const output = join(root, 'samples.json'); + + writeFileSync(input, '# bounded\n', 'utf8'); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + '0'.repeat(64), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 17bb601de3b1ff85b6159d3f88398f24fe4c1fd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:25:07 -0700 Subject: [PATCH 119/260] fix(perf): redact markdown module path errors --- benchmarks/measure-markdown.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 815630f7..e3458566 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -198,7 +198,12 @@ function resolveLocalModule(pathOrUrl) { } catch { throw new Error('Measured Markdown module must be a local regular file.'); } - const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } if ( metadata === undefined || metadata.isSymbolicLink() || @@ -206,7 +211,11 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured Markdown module must be a local regular file.'); } - return realpathSync(resolvedPath); + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } } function measuredModuleSha256(modulePath) { From 2d22587fde08312dd2032e50252672f117ea25cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:05:59 -0700 Subject: [PATCH 120/260] test(perf): reproduce revision module path leak --- ...asurementModulePathPrivacyContract.test.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts diff --git a/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..814f35c6 --- /dev/null +++ b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision benchmark module-path privacy contract', () => { + it('redacts filesystem details when module-path resolution crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-module-path-')); + const input = join(root, 'small.json'); + const privateSentinel = 'tenant-private-revision-module-parent'; + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'packed-revision-evidence.mjs'); + const output = join(root, 'samples.json'); + + try { + writeSyntheticEnvelope(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 054781216ca33d227b2a681a527e513789fed5b6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:08:36 -0700 Subject: [PATCH 121/260] fix(perf): redact revision module path errors --- benchmarks/measure-revision-evidence.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 3c413590..e2ed197b 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -180,7 +180,12 @@ function resolveLocalModule(pathOrUrl) { } catch { throw new Error('Measured revision module must be a local regular file.'); } - const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } if ( metadata === undefined || metadata.isSymbolicLink() || @@ -188,7 +193,11 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured revision module must be a local regular file.'); } - return realpathSync(resolvedPath); + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } } function measuredModuleSha256(modulePath) { From cecef7d4f042c8f5a6b32e8f5aa5bc1a6657a832 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:09:35 -0700 Subject: [PATCH 122/260] test(perf): reproduce revision input path leak --- ...easurementInputPathPrivacyContract.test.ts | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts diff --git a/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..fee61353 --- /dev/null +++ b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts @@ -0,0 +1,60 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +describe('revision benchmark input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-input-path-')); + const privateSentinel = 'tenant-private-revision-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-envelope.json'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From f412c50bc5d3ee95f236fc5870cb1543e2cd7bb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:10:33 -0700 Subject: [PATCH 123/260] fix(perf): redact revision file-open path errors --- benchmarks/measure-revision-evidence.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index e2ed197b..99286baa 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -102,7 +102,12 @@ function resolveArguments(argv) { } function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } if ( pathMetadata === undefined || pathMetadata.isSymbolicLink() || @@ -111,7 +116,12 @@ function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversize throw new Error(invalidFileMessage); } - const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From df1ac908bc04dfa6b5e290587b9f15c5ddce832a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:12:46 -0700 Subject: [PATCH 124/260] test(perf): reproduce existing-output alias path leaks --- ...kExistingOutputPathPrivacyContract.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts diff --git a/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts new file mode 100644 index 00000000..0555b3db --- /dev/null +++ b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const cases = [ + { + name: 'Markdown', + script: 'benchmarks/measure-markdown.mjs', + expectedError: 'Markdown benchmark input must be a regular non-symlink file.', + }, + { + name: 'revision', + script: 'benchmarks/measure-revision-evidence.mjs', + expectedError: 'Revision benchmark input must be a regular non-symlink file.', + }, +] as const; + +describe('benchmark existing-output path privacy contract', () => { + for (const testCase of cases) { + it(`redacts ${testCase.name} input paths before existing-output alias checks`, () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-alias-path-')); + const privateSentinel = `tenant-private-${testCase.name.toLowerCase()}-input-parent`; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-input'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(output, '{}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + resolve(process.cwd(), testCase.script), + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(testCase.expectedError); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + } +}); From 9488aad6098bf85d72485e44a0c553f7dbadcdba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:14:11 -0700 Subject: [PATCH 125/260] fix(perf): validate Markdown input before alias stat --- benchmarks/measure-markdown.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e3458566..8d0549b5 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -310,13 +310,13 @@ function writeMeasurementOutput(path, content) { async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedMarkdown(args.inputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) ) { throw new Error('Markdown benchmark output must not overwrite its input.'); } - const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); if ( modulePath === args.outputPath || From d2a7a1f5bda57c808a5514f975a5a3045065da7d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:17:01 -0700 Subject: [PATCH 126/260] fix(perf): validate revision input before alias stat --- benchmarks/measure-revision-evidence.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 99286baa..b0064e60 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -321,13 +321,13 @@ async function runMeasuredRevision(createRevisionEvidence, source) { async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedEnvelopeBytes(args.inputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) ) { throw new Error('Revision benchmark output must not overwrite its input.'); } - const source = readBoundedEnvelopeBytes(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); if ( modulePath === args.outputPath || From 82155d37b8226fe0052dde3c87c0dfc7c8940053 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:25:04 -0700 Subject: [PATCH 127/260] test(perf): reproduce summary input path leak --- ...nceSummaryInputPathPrivacyContract.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/performanceSummaryInputPathPrivacyContract.test.ts diff --git a/src/performanceSummaryInputPathPrivacyContract.test.ts b/src/performanceSummaryInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..64ca6050 --- /dev/null +++ b/src/performanceSummaryInputPathPrivacyContract.test.ts @@ -0,0 +1,37 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const summarizer = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-path-')); + const privateSentinel = 'tenant-private-summary-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'samples.json'); + const output = join(root, 'summary'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [summarizer, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 463f100d3cf7f3d24b560e3632decc75b9526d27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:09:29 -0700 Subject: [PATCH 128/260] fix(perf): redact summary input inspection failures --- benchmarks/summarize-samples.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index c2ff872e..9193d97e 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -60,8 +60,16 @@ function resolveArguments(argv) { }); } +function inspectSampleInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark sample input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectSampleInputPath(path); if (pathMetadata?.isSymbolicLink()) { throw new Error( 'Benchmark sample input must be a regular non-symlink file.', From f4400a6023f35f46744ef6d43b416ad9910db1a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:12:00 -0700 Subject: [PATCH 129/260] test(perf): expose retained-memory input path leakage --- src/performanceMemorySettlingContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index 93819962..63f26dce 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -133,6 +133,38 @@ describe('retained-memory settling evidence contract', () => { } }); + it('redacts an input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-path-privacy-')); + try { + const privateMarker = 'tenant-private-memory-evidence-parent'; + const parentPath = join(root, privateMarker); + writeFileSync(parentPath, 'not-a-directory', 'utf8'); + const inputPath = join(parentPath, 'memory-evidence.json'); + const result = spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + '50', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); try { From 5d1a62e9b376d8bba870efe03d3c1281c006c458 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:14:13 -0700 Subject: [PATCH 130/260] fix(perf): redact retained-memory input inspection failures --- benchmarks/analyze-memory-settling.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 9e60c947..f15d3d83 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -73,8 +73,16 @@ function resolveArguments(argv) { }); } +function inspectEvidenceInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Memory settling evidence input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectEvidenceInputPath(path); if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { throw new Error( 'Memory settling evidence input must be a regular non-symlink file.', From 6a6ee594e94db76295ca1888648b761f740374c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:16:13 -0700 Subject: [PATCH 131/260] test(perf): expose comparator input path leakage --- ...rmanceRegressionComparatorContract.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index dfeab91e..d51ae9f1 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -244,6 +244,47 @@ describe('benchmark regression comparator contract', () => { } }); + it('redacts a summary input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-path-privacy-')); + try { + const privateMarker = 'tenant-private-performance-baseline-parent'; + const privateParentPath = join(root, privateMarker); + const baselinePath = join(privateParentPath, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(privateParentPath, 'not-a-directory', 'utf8'); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 00bdb5b6a378f3582daf26f41f9652027b1f3b95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:18:00 -0700 Subject: [PATCH 132/260] fix(perf): redact comparator input inspection failures --- benchmarks/compare-summaries.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 4b891f14..a5c252da 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -86,8 +86,16 @@ function resolveArguments(argv) { }); } +function inspectSummaryInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectSummaryInputPath(path); if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { throw new Error( 'Benchmark summary input must be a regular non-symlink file.', From 45c6f41c15ddb90b674f31170bf402c29bcfedcb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:21:18 -0700 Subject: [PATCH 133/260] test(perf): expose Office fixture output symlink overwrite --- src/performanceOfficeFixtureContract.test.ts | 27 +++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index ebc1fa2d..1dfea952 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -144,4 +151,22 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of overwriting a file through an Office fixture output symlink', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e4d3478f1d4eb03954ac37c9a5d07802190f6bca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 13:05:56 -0700 Subject: [PATCH 134/260] fix(perf): fail closed on Office fixture output symlinks --- benchmarks/generate-office-fixtures.mjs | 58 ++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 40a512df..d09d4d36 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -1,5 +1,13 @@ import { createHash } from 'node:crypto'; -import { mkdirSync, writeFileSync } from 'node:fs'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const DOCX_PROFILE_PAGES = Object.freeze({ @@ -15,6 +23,12 @@ const PPTX_PROFILE_SLIDES = Object.freeze({ const EXCEL_MAX_COLUMNS = 16_384; const MULTILINGUAL_PARAGRAPH = 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; +const WRITE_NOFOLLOW = + constants.O_WRONLY | + constants.O_CREAT | + constants.O_TRUNC | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); @@ -154,10 +168,45 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } +function writeOutputFile(outputPath, bytes) { + let pathMetadata; + try { + pathMetadata = lstatSync(outputPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output path could not be inspected.'); + } + if (pathMetadata?.isSymbolicLink()) { + throw new Error('Office fixture output must be a regular non-symlink file.'); + } + if (pathMetadata !== undefined && !pathMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + + let descriptor; + try { + descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); + if (!fstatSync(descriptor).isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + writeFileSync(descriptor, bytes); + } catch (error) { + if ( + error instanceof Error && + (error.message === 'Office fixture output must be a regular file.' || + error.message === 'Office fixture output must be a regular non-symlink file.') + ) { + throw error; + } + throw new Error('Office fixture output could not be written safely.'); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + function writeFixture(outputDirectory, fileName, request, units) { const body = `${JSON.stringify(request, null, 2)}\n`; const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, fileName), bytes); + writeOutputFile(resolve(outputDirectory, fileName), bytes); return Object.freeze({ units, bytes: bytes.byteLength, @@ -212,8 +261,7 @@ const manifest = Object.freeze({ pptx: Object.freeze(pptx), }), }); -writeFileSync( +writeOutputFile( resolve(outputDirectory, 'manifest.json'), - `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', + Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), ); From 5fe6023c927e8944327ca4452e87caa0c464644d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 15:06:29 -0700 Subject: [PATCH 135/260] test(perf): expose Office fixture output directory symlinks --- src/performanceOfficeFixtureContract.test.ts | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index 1dfea952..39abb092 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, @@ -169,4 +170,41 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-dir-')); + const victimDirectory = join(root, 'buyer-owned'); + const outputDirectory = join(root, 'output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output ancestor', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-ancestor-')); + const victimDirectory = join(root, 'buyer-owned-parent'); + const linkedParent = join(root, 'linked-parent'); + const outputDirectory = join(linkedParent, 'nested-output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, linkedParent, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 7983e66cea4264a04988117459fe202c2ce89663 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 15:10:29 -0700 Subject: [PATCH 136/260] fix(perf): reject Office fixture output directory symlinks --- benchmarks/generate-office-fixtures.mjs | 33 +++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index d09d4d36..9a77b832 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -8,7 +8,7 @@ import { openSync, writeFileSync, } from 'node:fs'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; const DOCX_PROFILE_PAGES = Object.freeze({ small: 2, @@ -29,6 +29,8 @@ const WRITE_NOFOLLOW = constants.O_TRUNC | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); +const OUTPUT_DIRECTORY_ERROR = + 'Office fixture output directory must be a non-symlink directory.'; function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); @@ -168,6 +170,27 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputDirectoryAncestors(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function writeOutputFile(outputPath, bytes) { let pathMetadata; try { @@ -215,7 +238,13 @@ function writeFixture(outputDirectory, fileName, request, units) { } const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); -mkdirSync(outputDirectory, { recursive: true }); +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); +try { + mkdirSync(outputDirectory, { recursive: true }); +} catch { + throw new Error('Office fixture output directory could not be prepared.'); +} +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); const docx = {}; for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { From 31dee74f8dcf7b1383e7ea95953754f08c8adb7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:03:44 -0700 Subject: [PATCH 137/260] test(perf): reject hard-linked Office fixture outputs --- src/performanceOfficeFixtureContract.test.ts | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index 39abb092..c55b36dd 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -1,5 +1,6 @@ import { execFileSync } from 'node:child_process'; import { + linkSync, mkdirSync, mkdtempSync, readFileSync, @@ -171,6 +172,24 @@ describe('deterministic synthetic Office performance fixtures', () => { } }); + it('fails closed instead of overwriting a multiply linked Office fixture output', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-hardlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'buyer-owned.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + linkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { if (process.platform === 'win32') return; @@ -207,4 +226,4 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); -}); +}); \ No newline at end of file From 233f81e26827b83f6961fccbd07f44bb1db80181 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:06:19 -0700 Subject: [PATCH 138/260] fix(perf): reject hard-linked Office fixture outputs --- benchmarks/generate-office-fixtures.mjs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 9a77b832..e3264cc4 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -3,6 +3,7 @@ import { closeSync, constants, fstatSync, + ftruncateSync, lstatSync, mkdirSync, openSync, @@ -26,7 +27,6 @@ const MULTILINGUAL_PARAGRAPH = const WRITE_NOFOLLOW = constants.O_WRONLY | constants.O_CREAT | - constants.O_TRUNC | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); const OUTPUT_DIRECTORY_ERROR = @@ -208,15 +208,21 @@ function writeOutputFile(outputPath, bytes) { let descriptor; try { descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); - if (!fstatSync(descriptor).isFile()) { + const descriptorMetadata = fstatSync(descriptor); + if (!descriptorMetadata.isFile()) { throw new Error('Office fixture output must be a regular file.'); } + if (descriptorMetadata.nlink !== 1) { + throw new Error('Office fixture output must not be multiply linked.'); + } + ftruncateSync(descriptor, 0); writeFileSync(descriptor, bytes); } catch (error) { if ( error instanceof Error && (error.message === 'Office fixture output must be a regular file.' || - error.message === 'Office fixture output must be a regular non-symlink file.') + error.message === 'Office fixture output must be a regular non-symlink file.' || + error.message === 'Office fixture output must not be multiply linked.') ) { throw error; } @@ -293,4 +299,4 @@ const manifest = Object.freeze({ writeOutputFile( resolve(outputDirectory, 'manifest.json'), Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), -); +); \ No newline at end of file From cdd9e3492d0e459b813108d48118def960ccfc93 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:10:54 -0700 Subject: [PATCH 139/260] test(perf): expose summary output hard-link overwrite --- ...easurementStatisticsOutputHardlink.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/performanceMeasurementStatisticsOutputHardlink.test.ts diff --git a/src/performanceMeasurementStatisticsOutputHardlink.test.ts b/src/performanceMeasurementStatisticsOutputHardlink.test.ts new file mode 100644 index 00000000..75e18c00 --- /dev/null +++ b/src/performanceMeasurementStatisticsOutputHardlink.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'small', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output hard-link safety', () => { + it('fails closed before truncating an unrelated hard-linked output target', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-output-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const sentinel = join(root, 'buyer-owned.txt'); + const summaryJson = join(output, 'summary.json'); + + try { + writeInput(input); + mkdirSync(output); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, summaryJson); + const originalSentinel = readFileSync(sentinel, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must not be multiply linked.', + ); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 36d88306e52e8037f2e87bd37c450b489e40d9e1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:14:12 -0700 Subject: [PATCH 140/260] fix(perf): reject hard-linked summary outputs --- benchmarks/summarize-samples.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 9193d97e..a7330965 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -249,7 +249,7 @@ function formatSummary(summary) { `runtime_id=${summary.runtimeId}`, `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, - `percentile_method=${summary.percentileMethod}`, + `percentile_method=nearest-rank`, `minimum=${summary.minimum}`, `p50=${summary.p50}`, `p75=${summary.p75}`, @@ -271,6 +271,11 @@ function assertRegularOutputDestination(path) { if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } + if (metadata !== undefined && metadata.nlink !== 1) { + throw new Error( + 'Benchmark summary output paths must not be multiply linked.', + ); + } } function inspectOutputDirectoryComponent(path) { From 22223643b70ae681c4f2180d9fde8b14f032f8cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:15:07 -0700 Subject: [PATCH 141/260] chore(perf): preserve summary metadata formatting --- benchmarks/summarize-samples.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index a7330965..3fa1c787 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -249,7 +249,7 @@ function formatSummary(summary) { `runtime_id=${summary.runtimeId}`, `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, - `percentile_method=nearest-rank`, + `percentile_method=${summary.percentileMethod}`, `minimum=${summary.minimum}`, `p50=${summary.p50}`, `p75=${summary.p75}`, From 3ddee5a64328829f5eb34766684434542616ef6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:18:31 -0700 Subject: [PATCH 142/260] fix(perf): preserve summary alias diagnostics --- benchmarks/summarize-samples.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 3fa1c787..b98d2b87 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -271,6 +271,10 @@ function assertRegularOutputDestination(path) { if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } +} + +function assertSingleLinkOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); if (metadata !== undefined && metadata.nlink !== 1) { throw new Error( 'Benchmark summary output paths must not be multiply linked.', @@ -346,6 +350,8 @@ function main() { if (refersToSameFile(summaryJsonPath, summaryTextPath)) { throw new Error('Benchmark summary outputs must be distinct files.'); } + assertSingleLinkOutputDestination(summaryJsonPath); + assertSingleLinkOutputDestination(summaryTextPath); const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 399a67a17fdc271fa87dd6dd27b839d723014b61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:21:50 -0700 Subject: [PATCH 143/260] test(perf): expose producer output hard-link overwrite --- ...eMeasurementProducerOutputHardlink.test.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 src/performanceMeasurementProducerOutputHardlink.test.ts diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts new file mode 100644 index 00000000..e3210db3 --- /dev/null +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -0,0 +1,128 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + linkSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const markdownScript = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const revisionScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const sourceCommitSha = 'a'.repeat(40); +const runtimeId = 'node-22.18.0'; +const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; + +function sha256(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +function commonArguments( + input: string, + module: string, + moduleSha256: string, + output: string, +): string[] { + return [ + '--input', + input, + '--module', + module, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + moduleSha256, + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ]; +} + +function expectHardlinkFailure( + script: string, + args: string[], + sentinel: string, + expectedMessage: string, +): void { + const originalSentinel = readFileSync(sentinel, 'utf8'); + const result = spawnSync(process.execPath, [script, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedMessage); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); +} + +describe('benchmark producer output hard-link safety', () => { + it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectHardlinkFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Markdown benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-hardlink-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectHardlinkFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Revision benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 1b7da55909d83d9fdbf68f28e0e197577e437707 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:24:56 -0700 Subject: [PATCH 144/260] fix(perf): reject hard-linked Markdown output --- benchmarks/measure-markdown.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 8d0549b5..68f1cae1 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -365,6 +365,9 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Markdown benchmark output must not be multiply linked.'); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From a48c1d40947451550a7392ee889ab2c23d51702b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:25:40 -0700 Subject: [PATCH 145/260] fix(perf): reject hard-linked revision output --- benchmarks/measure-revision-evidence.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index b0064e60..a209e3e0 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -374,6 +374,9 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Revision benchmark output must not be multiply linked.'); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 2f94a6b7a551bc4b977ff36e5f20b3230522a2af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:10:46 -0700 Subject: [PATCH 146/260] test(perf): require revision evidence in single-command suite --- ...formanceSingleCommandSuiteContract.test.ts | 153 +++++++++++++----- 1 file changed, 114 insertions(+), 39 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 611fb419..ac90dfbd 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -25,17 +25,24 @@ afterEach(() => { }); function benchmarkArguments( - inputPath: string, - modulePath: string, - artifactSha256: string, + markdownInputPath: string, + markdownModulePath: string, + markdownArtifactSha256: string, + revisionInputPath: string, + revisionModulePath: string, + revisionArtifactSha256: string, outputDirectory: string, ): string[] { return [ suitePath, '--input', - inputPath, + markdownInputPath, '--module', - modulePath, + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, '--profile', 'small', '--samples', @@ -43,7 +50,9 @@ function benchmarkArguments( '--source-commit-sha', 'a'.repeat(40), '--artifact-sha256', - artifactSha256, + markdownArtifactSha256, + '--revision-artifact-sha256', + revisionArtifactSha256, '--runtime-id', 'node-22.0.0', '--reference-hardware-id', @@ -54,34 +63,62 @@ function benchmarkArguments( } function writeBenchmarkInputs(directory: string): { - artifactSha256: string; - inputPath: string; - modulePath: string; + markdownArtifactSha256: string; + markdownInputPath: string; + markdownModulePath: string; + revisionArtifactSha256: string; + revisionInputPath: string; + revisionModulePath: string; } { - const inputPath = join(directory, 'input.md'); - const modulePath = join(directory, 'measured.mjs'); - const moduleSource = + const markdownInputPath = join(directory, 'input.md'); + const markdownModulePath = join(directory, 'markdown-measured.mjs'); + const markdownModuleSource = "export function markdownToHtml(source) { return `

${source}

`; }\n"; - writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); - writeFileSync(modulePath, moduleSource, 'utf8'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const revisionModulePath = join(directory, 'revision-measured.mjs'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + writeFileSync(markdownInputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + return { - artifactSha256: createHash('sha256').update(moduleSource).digest('hex'), - inputPath, - modulePath, + markdownArtifactSha256: createHash('sha256') + .update(markdownModuleSource) + .digest('hex'), + markdownInputPath, + markdownModulePath, + revisionArtifactSha256: createHash('sha256') + .update(revisionModuleSource) + .digest('hex'), + revisionInputPath, + revisionModulePath, }; } describe('single-command benchmark suite contract', () => { - it('measures and summarizes one deterministic Markdown profile with one command', () => { + it('measures and summarizes Markdown serialization and revision evidence with one command', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); const outputDirectory = join(directory, 'evidence'); - const { artifactSha256, inputPath, modulePath } = - writeBenchmarkInputs(directory); + const inputs = writeBenchmarkInputs(directory); const output = execFileSync( process.execPath, - benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), { cwd: repositoryRoot, encoding: 'utf8', @@ -93,34 +130,64 @@ describe('single-command benchmark suite contract', () => { expect(JSON.parse(output.trim())).toEqual({ contractVersion: 1, documentProfile: 'small', - samples: 'samples.json', + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', status: 'completed', - summaryJson: 'summary/summary.json', - summaryText: 'summary/summary.txt', }); - const samples = JSON.parse( - readFileSync(join(outputDirectory, 'samples.json'), 'utf8'), + const markdownSamples = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; - expect(samples.benchmarkId).toBe('markdown-serialization-small'); - expect(samples.documentProfile).toBe('small'); - expect(samples.samples).toHaveLength(2); + expect(markdownSamples.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSamples.documentProfile).toBe('small'); + expect(markdownSamples.samples).toHaveLength(2); - const summary = JSON.parse( - readFileSync(join(outputDirectory, 'summary', 'summary.json'), 'utf8'), + const markdownSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.json'), + 'utf8', + ), ) as { benchmarkId?: unknown; documentProfile?: unknown }; - expect(summary.benchmarkId).toBe('markdown-serialization-small'); - expect(summary.documentProfile).toBe('small'); + expect(markdownSummary.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSummary.documentProfile).toBe('small'); expect( - readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.txt'), + 'utf8', + ), ).toContain('markdown-serialization-small'); + + const revisionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'revision', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(revisionSamples.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSamples.documentProfile).toBe('small'); + expect(revisionSamples.samples).toHaveLength(2); + + const revisionSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(revisionSummary.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('revision-evidence-small'); }); it('fails closed before writing evidence through a symlink output directory', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); - const { artifactSha256, inputPath, modulePath } = - writeBenchmarkInputs(directory); + const inputs = writeBenchmarkInputs(directory); const actualOutputDirectory = join(directory, 'outside-target'); const outputDirectory = join(directory, 'evidence-link'); mkdirSync(actualOutputDirectory); @@ -132,7 +199,15 @@ describe('single-command benchmark suite contract', () => { const result = spawnSync( process.execPath, - benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), { cwd: repositoryRoot, encoding: 'utf8', @@ -145,7 +220,7 @@ describe('single-command benchmark suite contract', () => { expect(result.stderr).toBe( 'Benchmark suite output directory must be a non-symlink directory.\n', ); - expect(existsSync(join(actualOutputDirectory, 'samples.json'))).toBe(false); - expect(existsSync(join(actualOutputDirectory, 'summary'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'revision'))).toBe(false); }); }); From 49baff63b57411dd70239730999766673419a617 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:11:21 -0700 Subject: [PATCH 147/260] feat(perf): compose revision evidence in current benchmark suite --- benchmarks/run-current-suite.mjs | 118 ++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 5d9471b2..58c07229 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -8,10 +8,13 @@ const repositoryRoot = resolve(benchmarkDirectory, '..'); const expectedFlags = Object.freeze([ '--input', '--module', + '--revision-input', + '--revision-module', '--profile', '--samples', '--source-commit-sha', '--artifact-sha256', + '--revision-artifact-sha256', '--runtime-id', '--reference-hardware-id', '--output', @@ -26,14 +29,47 @@ function resolveArguments(argv) { expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) ) { throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + 'Usage: node benchmarks/run-current-suite.mjs --input --module --revision-input --revision-module --profile --samples --source-commit-sha --artifact-sha256 --revision-artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const sharedArguments = Object.freeze([ + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]); + return Object.freeze({ - documentProfile: argv[5], - forwardedArguments: Object.freeze(argv.slice(0, -2)), - outputDirectory: resolve(argv[17]), + documentProfile: values['--profile'], + markdownArguments: Object.freeze([ + '--input', + values['--input'], + '--module', + values['--module'], + ...sharedArguments, + '--artifact-sha256', + values['--artifact-sha256'], + ]), + revisionArguments: Object.freeze([ + '--input', + values['--revision-input'], + '--module', + values['--revision-module'], + ...sharedArguments, + '--artifact-sha256', + values['--revision-artifact-sha256'], + ]), + outputDirectory: resolve(values['--output']), }); } @@ -103,31 +139,79 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { } } -function main(argv) { - const args = resolveArguments(argv); - prepareOutputDirectory(args.outputDirectory); - const samplesPath = resolve(args.outputDirectory, 'samples.json'); - const summaryDirectory = resolve(args.outputDirectory, 'summary'); - +function runMeasurementAndSummary({ + measurementScript, + measurementArguments, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { runBoundedNodeScript( - 'measure-markdown.mjs', - [...args.forwardedArguments, '--output', samplesPath], - 'Benchmark suite measurement failed.', + measurementScript, + [...measurementArguments, '--output', samplesPath], + measurementFailure, ); runBoundedNodeScript( 'summarize-samples.mjs', ['--input', samplesPath, '--output', summaryDirectory], - 'Benchmark suite summary failed.', + summaryFailure, ); +} + +function main(argv) { + const args = resolveArguments(argv); + prepareOutputDirectory(args.outputDirectory); + + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: args.markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: args.revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); process.stdout.write( `${JSON.stringify({ contractVersion: 1, documentProfile: args.documentProfile, - samples: 'samples.json', + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', status: 'completed', - summaryJson: 'summary/summary.json', - summaryText: 'summary/summary.txt', })}\n`, ); } From 2614a79480597f7fe5b77488b2738201289b19c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:13:38 -0700 Subject: [PATCH 148/260] fix(perf): preserve measured-script argument order --- benchmarks/run-current-suite.mjs | 34 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 58c07229..d28b5bd2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -36,18 +36,6 @@ function resolveArguments(argv) { const values = Object.fromEntries( expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), ); - const sharedArguments = Object.freeze([ - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]); return Object.freeze({ documentProfile: values['--profile'], @@ -56,18 +44,36 @@ function resolveArguments(argv) { values['--input'], '--module', values['--module'], - ...sharedArguments, + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], '--artifact-sha256', values['--artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], ]), revisionArguments: Object.freeze([ '--input', values['--revision-input'], '--module', values['--revision-module'], - ...sharedArguments, + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], '--artifact-sha256', values['--revision-artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], ]), outputDirectory: resolve(values['--output']), }); From 7ae80074210e7660ba28cdce89e5e4400bf022d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:16:55 -0700 Subject: [PATCH 149/260] test(perf): keep suite symlink regression on current CLI --- src/performanceMeasurementSuiteAncestorSymlink.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts index 7b61c4b5..5ea36b6d 100644 --- a/src/performanceMeasurementSuiteAncestorSymlink.test.ts +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -30,6 +30,10 @@ describe('benchmark suite output path ancestry', () => { join(root, 'unused.md'), '--module', join(root, 'unused.mjs'), + '--revision-input', + join(root, 'unused-envelope.json'), + '--revision-module', + join(root, 'unused-revision.mjs'), '--profile', 'large', '--samples', @@ -38,6 +42,8 @@ describe('benchmark suite output path ancestry', () => { 'a'.repeat(40), '--artifact-sha256', 'b'.repeat(64), + '--revision-artifact-sha256', + 'c'.repeat(64), '--runtime-id', 'node-22.18.0', '--reference-hardware-id', From eb6e8acf1145077ea2e2b40ed72a9660577dfcdf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:20:34 -0700 Subject: [PATCH 150/260] test(perf): require atomic suite evidence publication --- ...formanceSingleCommandSuiteContract.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index ac90dfbd..c87a73b9 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -184,6 +184,49 @@ describe('single-command benchmark suite contract', () => { ).toContain('revision-evidence-small'); }); + it('removes partial suite evidence when a downstream measurement fails', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const failingRevisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private benchmark failure'); }\n"; + writeFileSync( + inputs.revisionModulePath, + failingRevisionModuleSource, + 'utf8', + ); + const failingRevisionArtifactSha256 = createHash('sha256') + .update(failingRevisionModuleSource) + .digest('hex'); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + failingRevisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite revision measurement failed.\n', + ); + expect(existsSync(outputDirectory)).toBe(false); + }); + it('fails closed before writing evidence through a symlink output directory', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); From 0af333b74ad9ec619025f7bbb743c6b0cdb75c8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:24:40 -0700 Subject: [PATCH 151/260] fix(perf): remove partial suite evidence on failure --- benchmarks/run-current-suite.mjs | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index d28b5bd2..275d53ac 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { lstatSync, mkdirSync } from 'node:fs'; +import { lstatSync, mkdirSync, rmSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -107,7 +107,7 @@ function prepareOutputDirectory(path) { if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } - return; + return false; } try { @@ -121,6 +121,15 @@ function prepareOutputDirectory(path) { if (created === undefined || !created.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } } function runBoundedNodeScript(scriptName, args, failureMessage) { @@ -165,10 +174,7 @@ function runMeasurementAndSummary({ ); } -function main(argv) { - const args = resolveArguments(argv); - prepareOutputDirectory(args.outputDirectory); - +function runSuite(args) { const markdownSamplesPath = resolve( args.outputDirectory, 'markdown', @@ -206,6 +212,20 @@ function main(argv) { measurementFailure: 'Benchmark suite revision measurement failed.', summaryFailure: 'Benchmark suite revision summary failed.', }); +} + +function main(argv) { + const args = resolveArguments(argv); + const createdOutputDirectory = prepareOutputDirectory(args.outputDirectory); + + try { + runSuite(args); + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(args.outputDirectory); + } + throw error; + } process.stdout.write( `${JSON.stringify({ From d609faa683c742dfd757dc7d54bd664c9c676901 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:07:12 -0700 Subject: [PATCH 152/260] test(perf): require packed artifact benchmark provenance --- ...ormancePackedArtifactSuiteContract.test.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 src/performancePackedArtifactSuiteContract.test.ts diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts new file mode 100644 index 00000000..db4933dc --- /dev/null +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -0,0 +1,146 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture(directory: string): { + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, 'package-source'); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, 'packed'); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + "export function markdownToHtml(source) { return `

${source}

`; }\n", + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +describe('packed artifact benchmark suite contract', () => { + it('binds one-command benchmark evidence to a packed npm artifact digest', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); + temporaryDirectories.push(directory); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const outputDirectory = join(directory, 'evidence'); + writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + 'utf8', + ); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + packed.tarballPath, + '--package-sha256', + packed.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ + contractVersion: 1, + packageName: '@contextualwisdomlab/cwl-editor', + packageVersion: '0.0.0-benchmark-fixture', + packageSha256: packed.packageSha256, + status: 'completed', + }); + }); +}); From a49ea9defd5e66eb467bd71e15b2d31eb8c6261e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:11:05 -0700 Subject: [PATCH 153/260] fix(perf): bind suite to packed npm artifact --- benchmarks/run-current-suite.mjs | 467 ++++++++++++++++++++++++++----- 1 file changed, 397 insertions(+), 70 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 275d53ac..64ed643e 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,11 +1,24 @@ +import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; -import { lstatSync, mkdirSync, rmSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); -const expectedFlags = Object.freeze([ +const legacyFlags = Object.freeze([ '--input', '--module', '--revision-input', @@ -19,66 +32,129 @@ const expectedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; const OUTPUT_DIRECTORY_ERROR = 'Benchmark suite output directory must be a non-symlink directory.'; -function resolveArguments(argv) { - if ( - argv.length !== expectedFlags.length * 2 || - expectedFlags.some((flag, index) => argv[index * 2] !== flag) || - expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) - ) { - throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --module --revision-input --revision-module --profile --samples --source-commit-sha --artifact-sha256 --revision-artifact-sha256 --runtime-id --reference-hardware-id --output ', - ); - } +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} - const values = Object.fromEntries( +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), ); +} +function sharedArguments(values) { return Object.freeze({ documentProfile: values['--profile'], - markdownArguments: Object.freeze([ - '--input', - values['--input'], - '--module', - values['--module'], - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]), - revisionArguments: Object.freeze([ - '--input', - values['--revision-input'], - '--module', - values['--revision-module'], - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--revision-artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]), + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], outputDirectory: resolve(values['--output']), }); } +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function resolveArguments(argv) { + if (matchesArguments(argv, packedFlags)) { + const values = valuesForArguments(argv, packedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + function inspectOutputDirectory(path) { try { return lstatSync(path, { throwIfNoEntry: false }); @@ -132,6 +208,215 @@ function removePartialOutputDirectory(path) { } } +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + function runBoundedNodeScript(scriptName, args, failureMessage) { const result = spawnSync( process.execPath, @@ -156,7 +441,7 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { function runMeasurementAndSummary({ measurementScript, - measurementArguments, + measurementArguments: argumentsList, samplesPath, summaryDirectory, measurementFailure, @@ -164,7 +449,7 @@ function runMeasurementAndSummary({ }) { runBoundedNodeScript( measurementScript, - [...measurementArguments, '--output', samplesPath], + [...argumentsList, '--output', samplesPath], measurementFailure, ); runBoundedNodeScript( @@ -174,7 +459,7 @@ function runMeasurementAndSummary({ ); } -function runSuite(args) { +function runSuite(args, markdownArguments, revisionArguments) { const markdownSamplesPath = resolve( args.outputDirectory, 'markdown', @@ -198,7 +483,7 @@ function runSuite(args) { runMeasurementAndSummary({ measurementScript: 'measure-markdown.mjs', - measurementArguments: args.markdownArguments, + measurementArguments: markdownArguments, samplesPath: markdownSamplesPath, summaryDirectory: markdownSummaryDirectory, measurementFailure: 'Benchmark suite Markdown measurement failed.', @@ -206,7 +491,7 @@ function runSuite(args) { }); runMeasurementAndSummary({ measurementScript: 'measure-revision-evidence.mjs', - measurementArguments: args.revisionArguments, + measurementArguments: revisionArguments, samplesPath: revisionSamplesPath, summaryDirectory: revisionSummaryDirectory, measurementFailure: 'Benchmark suite revision measurement failed.', @@ -214,31 +499,73 @@ function runSuite(args) { }); } +function suiteManifest(args, packageEvidence) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); +} + function main(argv) { - const args = resolveArguments(argv); - const createdOutputDirectory = prepareOutputDirectory(args.outputDirectory); + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; try { - runSuite(args); + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } } catch (error) { if (createdOutputDirectory) { - removePartialOutputDirectory(args.outputDirectory); + removePartialOutputDirectory(shared.outputDirectory); } throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } } process.stdout.write( - `${JSON.stringify({ - contractVersion: 1, - documentProfile: args.documentProfile, - markdownSamples: 'markdown/samples.json', - markdownSummaryJson: 'markdown/summary/summary.json', - markdownSummaryText: 'markdown/summary/summary.txt', - revisionSamples: 'revision/samples.json', - revisionSummaryJson: 'revision/summary/summary.json', - revisionSummaryText: 'revision/summary/summary.txt', - status: 'completed', - })}\n`, + `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, ); } From f0c9d8f5fb5ad498577c5b4fe2919a6a51dacf9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:13:55 -0700 Subject: [PATCH 154/260] test(perf): reject mislabeled benchmark runtime provenance --- ...ormancePackedArtifactSuiteContract.test.ts | 106 ++++++++++++------ 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index db4933dc..4d1f3949 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -15,6 +15,7 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -85,46 +86,59 @@ function createPackedBenchmarkFixture(directory: string): { }; } +function packedSuiteArguments(options: { + directory: string; + packageSha256: string; + runtimeId: string; + tarballPath: string; +}): string[] { + const markdownInputPath = join(options.directory, 'input.md'); + const revisionInputPath = join(options.directory, 'document-envelope.json'); + writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + 'utf8', + ); + return [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + options.tarballPath, + '--package-sha256', + options.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--runtime-id', + options.runtimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + join(options.directory, 'evidence'), + ]; +} + describe('packed artifact benchmark suite contract', () => { it('binds one-command benchmark evidence to a packed npm artifact digest', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); temporaryDirectories.push(directory); - const markdownInputPath = join(directory, 'input.md'); - const revisionInputPath = join(directory, 'document-envelope.json'); - const outputDirectory = join(directory, 'evidence'); - writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', - 'utf8', - ); const packed = createPackedBenchmarkFixture(directory); const result = spawnSync( process.execPath, - [ - suitePath, - '--input', - markdownInputPath, - '--revision-input', - revisionInputPath, - '--package-tarball', - packed.tarballPath, - '--package-sha256', - packed.packageSha256, - '--profile', - 'small', - '--samples', - '2', - '--source-commit-sha', - 'a'.repeat(40), - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }), { cwd: repositoryRoot, encoding: 'utf8', @@ -143,4 +157,32 @@ describe('packed artifact benchmark suite contract', () => { status: 'completed', }); }); + + it('rejects a runtime identifier that does not match the active Node process', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-runtime-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: 'node-0.0.0', + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite runtime ID must match the active Node runtime.\n', + ); + }); }); From 0c9f3f3e4e2c1c30e9ed0f3adc8469de13417602 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:18:27 -0700 Subject: [PATCH 155/260] fix(perf): attest active benchmark runtime --- benchmarks/run-current-suite.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 64ed643e..e63398cf 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -120,6 +120,12 @@ function resolveArguments(argv) { 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', ); } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } return Object.freeze({ mode: 'packed', shared: sharedArguments(values), From 5c8f6f8670c6906ffef648470c763202aa16ee11 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 07:28:35 -0700 Subject: [PATCH 156/260] test(perf): bind suite manifest to run provenance --- src/performanceSingleCommandSuiteContract.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index c87a73b9..0eae1cc5 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -130,6 +130,10 @@ describe('single-command benchmark suite contract', () => { expect(JSON.parse(output.trim())).toEqual({ contractVersion: 1, documentProfile: 'small', + sampleCount: 2, + sourceCommitSha: 'a'.repeat(40), + runtimeId: 'node-22.0.0', + referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', markdownSummaryText: 'markdown/summary/summary.txt', From 946c7150ad1bd7bcd8e30268bb7ec1d40f8a814e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 07:30:12 -0700 Subject: [PATCH 157/260] fix(perf): bind suite manifest to run provenance --- benchmarks/run-current-suite.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index e63398cf..4d9b912a 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -509,6 +509,10 @@ function suiteManifest(args, packageEvidence) { return Object.freeze({ contractVersion: 1, documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, ...(packageEvidence ?? {}), markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', From 413b0f392da15e1bf672f0177950a3654e5271af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 22:14:56 -0700 Subject: [PATCH 158/260] test(perf): attest packed suite run provenance --- src/performancePackedArtifactSuiteContract.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 4d1f3949..d0a8db8a 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -16,6 +16,8 @@ const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = 'a'.repeat(40); +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -115,18 +117,18 @@ function packedSuiteArguments(options: { '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + sourceCommitSha, '--runtime-id', options.runtimeId, '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, + referenceHardwareId, '--output', join(options.directory, 'evidence'), ]; } describe('packed artifact benchmark suite contract', () => { - it('binds one-command benchmark evidence to a packed npm artifact digest', () => { + it('binds one-command benchmark evidence to packed artifact and run provenance', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); temporaryDirectories.push(directory); const packed = createPackedBenchmarkFixture(directory); @@ -151,6 +153,11 @@ describe('packed artifact benchmark suite contract', () => { expect(result.stderr).toBe(''); expect(JSON.parse(result.stdout.trim())).toMatchObject({ contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha, + runtimeId: activeRuntimeId, + referenceHardwareId, packageName: '@contextualwisdomlab/cwl-editor', packageVersion: '0.0.0-benchmark-fixture', packageSha256: packed.packageSha256, From f136388a7909cf31305ff02f84324eb942d2ceed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 03:08:59 -0700 Subject: [PATCH 159/260] test(perf): bind benchmark source provenance to checkout --- ...ormancePackedArtifactSuiteContract.test.ts | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index d0a8db8a..35098ef8 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -16,7 +16,11 @@ const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; const activeRuntimeId = `node-${process.versions.node}`; -const sourceCommitSha = 'a'.repeat(40); +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; afterEach(() => { @@ -92,6 +96,7 @@ function packedSuiteArguments(options: { directory: string; packageSha256: string; runtimeId: string; + sourceCommitSha?: string; tarballPath: string; }): string[] { const markdownInputPath = join(options.directory, 'input.md'); @@ -117,7 +122,7 @@ function packedSuiteArguments(options: { '--samples', '2', '--source-commit-sha', - sourceCommitSha, + options.sourceCommitSha ?? sourceCommitSha, '--runtime-id', options.runtimeId, '--reference-hardware-id', @@ -192,4 +197,33 @@ describe('packed artifact benchmark suite contract', () => { 'Benchmark suite runtime ID must match the active Node runtime.\n', ); }); + + it('rejects source provenance that does not match the benchmark checkout', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-source-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + sourceCommitSha: '0'.repeat(40), + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit SHA must match the current benchmark checkout.\n', + ); + }); }); From 9876531b0688be8b974b010d37ebdd191cdfd389 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 03:14:16 -0700 Subject: [PATCH 160/260] fix(perf): verify benchmark source checkout provenance --- benchmarks/run-current-suite.mjs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 4d9b912a..d9717fc3 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -45,6 +45,7 @@ const packedFlags = Object.freeze([ '--output', ]); const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; @@ -111,6 +112,28 @@ function measurementArguments({ ]); } +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + function resolveArguments(argv) { if (matchesArguments(argv, packedFlags)) { const values = valuesForArguments(argv, packedFlags); @@ -126,6 +149,11 @@ function resolveArguments(argv) { 'Benchmark suite runtime ID must match the active Node runtime.', ); } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } return Object.freeze({ mode: 'packed', shared: sharedArguments(values), From 4a9661f1d0f6675473fe1c44a05949ff588c1083 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:12:07 -0700 Subject: [PATCH 161/260] fix(perf): reject dirty benchmark provenance --- benchmarks/run-current-suite-core.mjs | 617 ++++++++++++++++++ benchmarks/run-current-suite.mjs | 607 +---------------- benchmarks/source-checkout-provenance.mjs | 34 + ...ceSourceCheckoutProvenanceContract.test.ts | 40 ++ 4 files changed, 703 insertions(+), 595 deletions(-) create mode 100644 benchmarks/run-current-suite-core.mjs create mode 100644 benchmarks/source-checkout-provenance.mjs create mode 100644 src/performanceSourceCheckoutProvenanceContract.test.ts diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs new file mode 100644 index 00000000..d9717fc3 --- /dev/null +++ b/benchmarks/run-current-suite-core.mjs @@ -0,0 +1,617 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function sharedArguments(values) { + return Object.freeze({ + documentProfile: values['--profile'], + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], + outputDirectory: resolve(values['--output']), + }); +} + +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + +function resolveArguments(argv) { + if (matchesArguments(argv, packedFlags)) { + const values = valuesForArguments(argv, packedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (!existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return false; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectory(path); + if (created === undefined || !created.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function runMeasurementAndSummary({ + measurementScript, + measurementArguments: argumentsList, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { + runBoundedNodeScript( + measurementScript, + [...argumentsList, '--output', samplesPath], + measurementFailure, + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + summaryFailure, + ); +} + +function runSuite(args, markdownArguments, revisionArguments) { + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); +} + +function suiteManifest(args, packageEvidence) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); +} + +function main(argv) { + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; + try { + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(shared.outputDirectory); + } + throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } + + process.stdout.write( + `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index d9717fc3..cc87a585 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,610 +1,27 @@ -import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; -import { - closeSync, - constants, - fstatSync, - lstatSync, - mkdirSync, - mkdtempSync, - openSync, - readSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; + const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); -const legacyFlags = Object.freeze([ - '--input', - '--module', - '--revision-input', - '--revision-module', - '--profile', - '--samples', - '--source-commit-sha', - '--artifact-sha256', - '--revision-artifact-sha256', - '--runtime-id', - '--reference-hardware-id', - '--output', -]); -const packedFlags = Object.freeze([ - '--input', - '--revision-input', - '--package-tarball', - '--package-sha256', - '--profile', - '--samples', - '--source-commit-sha', - '--runtime-id', - '--reference-hardware-id', - '--output', -]); -const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; -const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; -const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; -const MAX_MODULE_BYTES = 16 * 1024 * 1024; -const READ_CHUNK_BYTES = 64 * 1024; -const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); -const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; -const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; -const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; -const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; -const OUTPUT_DIRECTORY_ERROR = - 'Benchmark suite output directory must be a non-symlink directory.'; - -function matchesArguments(argv, expectedFlags) { - return ( - argv.length === expectedFlags.length * 2 && - expectedFlags.every((flag, index) => argv[index * 2] === flag) && - expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) - ); -} - -function valuesForArguments(argv, expectedFlags) { - return Object.fromEntries( - expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), - ); -} - -function sharedArguments(values) { - return Object.freeze({ - documentProfile: values['--profile'], - sampleCount: values['--samples'], - sourceCommitSha: values['--source-commit-sha'], - runtimeId: values['--runtime-id'], - referenceHardwareId: values['--reference-hardware-id'], - markdownInputPath: values['--input'], - revisionInputPath: values['--revision-input'], - outputDirectory: resolve(values['--output']), - }); -} - -function measurementArguments({ - inputPath, - modulePath, - artifactSha256, - shared, -}) { - return Object.freeze([ - '--input', - inputPath, - '--module', - modulePath, - '--profile', - shared.documentProfile, - '--samples', - shared.sampleCount, - '--source-commit-sha', - shared.sourceCommitSha, - '--artifact-sha256', - artifactSha256, - '--runtime-id', - shared.runtimeId, - '--reference-hardware-id', - shared.referenceHardwareId, - ]); -} - -function currentCheckoutSha() { - const result = spawnSync('git', ['rev-parse', 'HEAD'], { - cwd: repositoryRoot, - encoding: 'utf8', - maxBuffer: 1024, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 10_000, - }); - const checkoutSha = result.stdout?.trim(); - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 || - !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') - ) { - throw new Error( - 'Benchmark suite source commit SHA could not be verified against the current checkout.', - ); - } - return checkoutSha; -} - -function resolveArguments(argv) { - if (matchesArguments(argv, packedFlags)) { - const values = valuesForArguments(argv, packedFlags); - const packageSha256 = values['--package-sha256']; - if (!SHA256_PATTERN.test(packageSha256)) { - throw new Error( - 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', - ); - } - const activeRuntimeId = `node-${process.versions.node}`; - if (values['--runtime-id'] !== activeRuntimeId) { - throw new Error( - 'Benchmark suite runtime ID must match the active Node runtime.', - ); - } - if (values['--source-commit-sha'] !== currentCheckoutSha()) { - throw new Error( - 'Benchmark suite source commit SHA must match the current benchmark checkout.', - ); - } - return Object.freeze({ - mode: 'packed', - shared: sharedArguments(values), - packageTarballPath: resolve(values['--package-tarball']), - packageSha256, - }); - } - - if (matchesArguments(argv, legacyFlags)) { - const values = valuesForArguments(argv, legacyFlags); - const shared = sharedArguments(values); - return Object.freeze({ - mode: 'module', - shared, - markdownArguments: measurementArguments({ - inputPath: shared.markdownInputPath, - modulePath: values['--module'], - artifactSha256: values['--artifact-sha256'], - shared, - }), - revisionArguments: measurementArguments({ - inputPath: shared.revisionInputPath, - modulePath: values['--revision-module'], - artifactSha256: values['--revision-artifact-sha256'], - shared, - }), - packageEvidence: null, - }); - } - - throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', - ); -} - -function inspectOutputDirectory(path) { - try { - return lstatSync(path, { throwIfNoEntry: false }); - } catch { - throw new Error('Benchmark suite output directory could not be inspected.'); - } -} - -function assertNoSymlinkDirectoryComponents(path) { - let current = path; - while (true) { - const metadata = inspectOutputDirectory(current); - if (metadata?.isSymbolicLink()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - const parent = dirname(current); - if (parent === current) return; - current = parent; - } -} - -function prepareOutputDirectory(path) { - assertNoSymlinkDirectoryComponents(path); - const existing = inspectOutputDirectory(path); - if (existing !== undefined) { - if (!existing.isDirectory()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - return false; - } - - try { - mkdirSync(path, { recursive: true }); - } catch { - throw new Error('Benchmark suite output directory could not be prepared.'); - } - - assertNoSymlinkDirectoryComponents(path); - const created = inspectOutputDirectory(path); - if (created === undefined || !created.isDirectory()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - return true; -} - -function removePartialOutputDirectory(path) { - try { - rmSync(path, { recursive: true, force: true }); - } catch { - throw new Error('Benchmark suite partial evidence could not be removed.'); - } -} - -function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { - let pathMetadata; - try { - pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - } catch { - throw new Error(invalidMessage); - } - if ( - pathMetadata === undefined || - pathMetadata.isSymbolicLink() || - !pathMetadata.isFile() - ) { - throw new Error(invalidMessage); - } - - let descriptor; - try { - descriptor = openSync(path, READ_ONLY_NOFOLLOW); - } catch { - throw new Error(invalidMessage); - } - try { - const metadata = fstatSync(descriptor); - if (!metadata.isFile()) throw new Error(invalidMessage); - if (metadata.size > maximumBytes) throw new Error(oversizedMessage); - - const chunks = []; - let totalBytes = 0; - while (totalBytes <= maximumBytes) { - const remainingBudget = maximumBytes + 1 - totalBytes; - const chunk = Buffer.allocUnsafe( - Math.min(READ_CHUNK_BYTES, remainingBudget), - ); - const bytesRead = readSync( - descriptor, - chunk, - 0, - chunk.byteLength, - null, - ); - if (bytesRead === 0) break; - totalBytes += bytesRead; - if (totalBytes > maximumBytes) throw new Error(oversizedMessage); - chunks.push(chunk.subarray(0, bytesRead)); - } - return Buffer.concat(chunks, totalBytes); - } finally { - closeSync(descriptor); - } -} - -function packageTarballBytes(path) { - return readBoundedRegularFile( - path, - MAX_PACKAGE_BYTES, - 'Benchmark suite package tarball must be a regular non-symlink file.', - 'Benchmark suite package tarball exceeds the supported size.', - ); -} +const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); -function verifyPackageDigest(path, expectedSha256) { - const actualSha256 = createHash('sha256') - .update(packageTarballBytes(path)) - .digest('hex'); - if (actualSha256 !== expectedSha256) { - throw new Error('Benchmark suite package digest does not match the packed artifact.'); - } -} +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); -function runTar(argumentsList, maximumBytes, failureMessage) { - const result = spawnSync('tar', argumentsList, { + const result = spawnSync(process.execPath, [coreRunnerPath, ...argv], { cwd: repositoryRoot, - maxBuffer: maximumBytes, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 30_000, + stdio: 'inherit', + timeout: 600_000, }); - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 || - !Buffer.isBuffer(result.stdout) || - result.stdout.byteLength > maximumBytes - ) { - throw new Error(failureMessage); - } - return result.stdout; -} - -function decodeUtf8(bytes, failureMessage) { - try { - return new TextDecoder('utf-8', { fatal: true }).decode(bytes); - } catch { - throw new Error(failureMessage); - } -} - -function listPackedEntries(tarballPath) { - const listing = decodeUtf8( - runTar( - ['-tzf', tarballPath], - MAX_PACKAGE_INDEX_BYTES, - 'Benchmark suite package index could not be read.', - ), - 'Benchmark suite package index must be valid UTF-8.', - ); - return listing.split('\n').filter((entry) => entry.length > 0); -} - -function assertUniquePackedEntry(entries, expectedEntry) { - if (entries.filter((entry) => entry === expectedEntry).length !== 1) { - throw new Error('Benchmark suite package is missing a unique required artifact.'); - } -} - -function readPackedEntry(tarballPath, entry, maximumBytes) { - return runTar( - ['-xOzf', tarballPath, entry], - maximumBytes, - 'Benchmark suite package artifact could not be read.', - ); -} - -function parsePackageManifest(bytes) { - let manifest; - try { - manifest = JSON.parse( - decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), - ); - } catch (error) { - if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; - throw new Error('Benchmark suite package manifest must be valid JSON.'); - } - if ( - manifest === null || - typeof manifest !== 'object' || - Array.isArray(manifest) || - manifest.name !== EXPECTED_PACKAGE_NAME || - typeof manifest.version !== 'string' || - manifest.version.length === 0 || - manifest.version.length > 128 - ) { - throw new Error('Benchmark suite package identity is invalid.'); - } - return Object.freeze({ name: manifest.name, version: manifest.version }); -} - -function moduleSha256(bytes) { - return createHash('sha256').update(bytes).digest('hex'); -} - -function preparePackedBenchmarkModules(args) { - verifyPackageDigest(args.packageTarballPath, args.packageSha256); - const entries = listPackedEntries(args.packageTarballPath); - for (const entry of [ - PACKAGE_MANIFEST_ENTRY, - MARKDOWN_MODULE_ENTRY, - REVISION_MODULE_ENTRY, - ]) { - assertUniquePackedEntry(entries, entry); - } - - const manifestBytes = readPackedEntry( - args.packageTarballPath, - PACKAGE_MANIFEST_ENTRY, - MAX_PACKAGE_MANIFEST_BYTES, - ); - const manifest = parsePackageManifest(manifestBytes); - const markdownModuleBytes = readPackedEntry( - args.packageTarballPath, - MARKDOWN_MODULE_ENTRY, - MAX_MODULE_BYTES, - ); - const revisionModuleBytes = readPackedEntry( - args.packageTarballPath, - REVISION_MODULE_ENTRY, - MAX_MODULE_BYTES, - ); - verifyPackageDigest(args.packageTarballPath, args.packageSha256); - - const temporaryDirectory = mkdtempSync( - join(tmpdir(), 'inkspan-packed-benchmark-'), - ); - const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); - const revisionModulePath = join( - temporaryDirectory, - 'cwl-revision-evidence.mjs', - ); - try { - writeFileSync(markdownModulePath, markdownModuleBytes); - writeFileSync(revisionModulePath, revisionModuleBytes); - } catch { - rmSync(temporaryDirectory, { recursive: true, force: true }); - throw new Error('Benchmark suite package modules could not be prepared.'); - } - - return Object.freeze({ - temporaryDirectory, - markdownModulePath, - markdownArtifactSha256: moduleSha256(markdownModuleBytes), - revisionModulePath, - revisionArtifactSha256: moduleSha256(revisionModuleBytes), - packageEvidence: Object.freeze({ - packageName: manifest.name, - packageVersion: manifest.version, - packageSha256: args.packageSha256, - }), - }); -} - -function runBoundedNodeScript(scriptName, args, failureMessage) { - const result = spawnSync( - process.execPath, - [resolve(benchmarkDirectory, scriptName), ...args], - { - cwd: repositoryRoot, - encoding: 'utf8', - maxBuffer: 4 * 1024 * 1024, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 120_000, - }, - ); - - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 - ) { - throw new Error(failureMessage); - } -} - -function runMeasurementAndSummary({ - measurementScript, - measurementArguments: argumentsList, - samplesPath, - summaryDirectory, - measurementFailure, - summaryFailure, -}) { - runBoundedNodeScript( - measurementScript, - [...argumentsList, '--output', samplesPath], - measurementFailure, - ); - runBoundedNodeScript( - 'summarize-samples.mjs', - ['--input', samplesPath, '--output', summaryDirectory], - summaryFailure, - ); -} - -function runSuite(args, markdownArguments, revisionArguments) { - const markdownSamplesPath = resolve( - args.outputDirectory, - 'markdown', - 'samples.json', - ); - const markdownSummaryDirectory = resolve( - args.outputDirectory, - 'markdown', - 'summary', - ); - const revisionSamplesPath = resolve( - args.outputDirectory, - 'revision', - 'samples.json', - ); - const revisionSummaryDirectory = resolve( - args.outputDirectory, - 'revision', - 'summary', - ); - - runMeasurementAndSummary({ - measurementScript: 'measure-markdown.mjs', - measurementArguments: markdownArguments, - samplesPath: markdownSamplesPath, - summaryDirectory: markdownSummaryDirectory, - measurementFailure: 'Benchmark suite Markdown measurement failed.', - summaryFailure: 'Benchmark suite Markdown summary failed.', - }); - runMeasurementAndSummary({ - measurementScript: 'measure-revision-evidence.mjs', - measurementArguments: revisionArguments, - samplesPath: revisionSamplesPath, - summaryDirectory: revisionSummaryDirectory, - measurementFailure: 'Benchmark suite revision measurement failed.', - summaryFailure: 'Benchmark suite revision summary failed.', - }); -} - -function suiteManifest(args, packageEvidence) { - return Object.freeze({ - contractVersion: 1, - documentProfile: args.documentProfile, - sampleCount: Number(args.sampleCount), - sourceCommitSha: args.sourceCommitSha, - runtimeId: args.runtimeId, - referenceHardwareId: args.referenceHardwareId, - ...(packageEvidence ?? {}), - markdownSamples: 'markdown/samples.json', - markdownSummaryJson: 'markdown/summary/summary.json', - markdownSummaryText: 'markdown/summary/summary.txt', - revisionSamples: 'revision/samples.json', - revisionSummaryJson: 'revision/summary/summary.json', - revisionSummaryText: 'revision/summary/summary.txt', - status: 'completed', - }); -} - -function main(argv) { - const resolved = resolveArguments(argv); - const shared = resolved.shared; - let preparedPackage; - if (resolved.mode === 'packed') { - preparedPackage = preparePackedBenchmarkModules(resolved); - } - - const markdownArguments = - resolved.mode === 'packed' - ? measurementArguments({ - inputPath: shared.markdownInputPath, - modulePath: preparedPackage.markdownModulePath, - artifactSha256: preparedPackage.markdownArtifactSha256, - shared, - }) - : resolved.markdownArguments; - const revisionArguments = - resolved.mode === 'packed' - ? measurementArguments({ - inputPath: shared.revisionInputPath, - modulePath: preparedPackage.revisionModulePath, - artifactSha256: preparedPackage.revisionArtifactSha256, - shared, - }) - : resolved.revisionArguments; - const packageEvidence = - resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; - let createdOutputDirectory = false; - try { - createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); - runSuite(shared, markdownArguments, revisionArguments); - if (resolved.mode === 'packed') { - verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); - } - } catch (error) { - if (createdOutputDirectory) { - removePartialOutputDirectory(shared.outputDirectory); - } - throw error; - } finally { - if (preparedPackage !== undefined) { - rmSync(preparedPackage.temporaryDirectory, { - recursive: true, - force: true, - }); - } + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); } - process.stdout.write( - `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, - ); + process.exitCode = result.status ?? 1; } try { diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs new file mode 100644 index 00000000..e2362381 --- /dev/null +++ b/benchmarks/source-checkout-provenance.mjs @@ -0,0 +1,34 @@ +import { spawnSync } from 'node:child_process'; + +const MAX_STATUS_BYTES = 1024 * 1024; + +export function assertCleanSourceCheckout(repositoryRoot) { + const result = spawnSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout cleanliness could not be verified.', + ); + } + + if (result.stdout.length !== 0) { + throw new Error( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', + ); + } +} diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts new file mode 100644 index 00000000..52ed1e3f --- /dev/null +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -0,0 +1,40 @@ +import { spawnSync } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const dirtySentinelPath = resolve( + repositoryRoot, + `.inkspan-benchmark-dirty-provenance-${process.pid}`, +); + +afterEach(() => { + rmSync(dirtySentinelPath, { force: true }); +}); + +describe('benchmark source checkout provenance', () => { + it('rejects untracked source state before acquisition evidence can run', () => { + writeFileSync(dirtySentinelPath, 'untracked benchmark provenance sentinel\n'); + + const result = spawnSync(process.execPath, [suitePath], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', + ); + expect(result.stderr).not.toContain('Usage:'); + expect(result.stderr).not.toContain(dirtySentinelPath); + }); +}); From b376877eb971f0070956c3bf8ef8c88dc0b71150 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:14:43 -0700 Subject: [PATCH 162/260] test(perf): isolate checkout provenance regression --- ...ceSourceCheckoutProvenanceContract.test.ts | 83 +++++++++++++++---- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts index 52ed1e3f..94f37cc2 100644 --- a/src/performanceSourceCheckoutProvenanceContract.test.ts +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -1,31 +1,82 @@ -import { spawnSync } from 'node:child_process'; -import { rmSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); -const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); -const dirtySentinelPath = resolve( - repositoryRoot, - `.inkspan-benchmark-dirty-provenance-${process.pid}`, -); +const helperUrl = pathToFileURL( + resolve(repositoryRoot, 'benchmarks/source-checkout-provenance.mjs'), +).href; +const temporaryDirectories: string[] = []; + +const probe = ` +import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; +try { + assertCleanSourceCheckout(process.argv[1]); + process.stdout.write('clean\\n'); +} catch (error) { + process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); + process.exitCode = 1; +} +`; afterEach(() => { - rmSync(dirtySentinelPath, { force: true }); + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } }); -describe('benchmark source checkout provenance', () => { - it('rejects untracked source state before acquisition evidence can run', () => { - writeFileSync(dirtySentinelPath, 'untracked benchmark provenance sentinel\n'); +function createRepository(): string { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + temporaryDirectories.push(directory); + execFileSync('git', ['init', '--quiet'], { cwd: directory }); + execFileSync('git', ['config', 'user.email', 'test@example.invalid'], { + cwd: directory, + }); + execFileSync('git', ['config', 'user.name', 'Inkspan Test'], { + cwd: directory, + }); + writeFileSync(join(directory, 'tracked.txt'), 'committed\n'); + execFileSync('git', ['add', 'tracked.txt'], { cwd: directory }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture'], { cwd: directory }); + return directory; +} - const result = spawnSync(process.execPath, [suitePath], { +function probeCheckout(directory: string) { + return spawnSync( + process.execPath, + ['--input-type=module', '--eval', probe, directory], + { cwd: repositoryRoot, encoding: 'utf8', maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], timeout: 10_000, - }); + }, + ); +} + +describe('benchmark source checkout provenance', () => { + it('accepts a clean source checkout', () => { + const directory = createRepository(); + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stdout).toBe('clean\n'); + expect(result.stderr).toBe(''); + }); + + it('rejects untracked source state without disclosing paths', () => { + const directory = createRepository(); + const untrackedPath = join(directory, 'untracked-secret-name.txt'); + writeFileSync(untrackedPath, 'not part of the committed source\n'); + + const result = probeCheckout(directory); expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); @@ -34,7 +85,7 @@ describe('benchmark source checkout provenance', () => { expect(result.stderr).toBe( 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', ); - expect(result.stderr).not.toContain('Usage:'); - expect(result.stderr).not.toContain(dirtySentinelPath); + expect(result.stderr).not.toContain(untrackedPath); + expect(result.stderr).not.toContain('untracked-secret-name.txt'); }); }); From 8b7b6df06f4d3d8574460acdbd6993c5ee0a203e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:40:19 -0700 Subject: [PATCH 163/260] test(perf): require Office duration and RSS evidence --- office/tests/test_performance_measurement.py | 111 +++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 office/tests/test_performance_measurement.py diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py new file mode 100644 index 00000000..94c14d7d --- /dev/null +++ b/office/tests/test_performance_measurement.py @@ -0,0 +1,111 @@ +"""Contract tests for privacy-safe Office render performance evidence.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure repeated synthetic renders without copying document content into evidence.""" + + sentinel = "PRIVATE-BENCHMARK-PAYLOAD-SENTINEL" + request_path = tmp_path / "buyer-private-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "Synthetic benchmark fixture", + "blocks": [ + {"type": "heading", "level": 1, "text": "Synthetic heading"}, + {"type": "paragraph", "text": sentinel}, + ], + } + ), + encoding="utf-8", + ) + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(request_path), + "--profile", + "docx-small", + "--iterations", + "2", + "--reference-hardware", + "pytest-reference", + ], + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + evidence = json.loads(completed.stdout) + assert evidence["contractVersion"] == 1 + assert evidence["synthetic"] is True + assert evidence["operation"] == "office_render" + assert evidence["profile"] == "docx-small" + assert evidence["format"] == "docx" + assert evidence["iterations"] == 2 + assert evidence["referenceHardware"] == "pytest-reference" + assert len(evidence["fixtureSha256"]) == 64 + assert len(evidence["sourceSha"]) == 40 + assert evidence["runtime"]["implementation"] + assert evidence["runtime"]["python"] + assert evidence["runtime"]["platform"] + assert len(evidence["samples"]) == 2 + for sample in evidence["samples"]: + assert isinstance(sample["durationMs"], (int, float)) + assert sample["durationMs"] >= 0 + assert isinstance(sample["peakRssBytes"], int) + assert sample["peakRssBytes"] > 0 + assert evidence["summary"]["durationMs"]["p50"] >= 0 + assert evidence["summary"]["durationMs"]["p75"] >= 0 + assert evidence["summary"]["durationMs"]["p95"] >= 0 + assert evidence["summary"]["durationMs"]["max"] >= 0 + assert evidence["summary"]["peakRssBytes"]["max"] > 0 + + combined_output = completed.stdout + completed.stderr + assert sentinel not in combined_output + assert str(request_path) not in combined_output + + +def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( + tmp_path: Path, +) -> None: + """Reject impossible benchmark work before inspecting the caller-selected request path.""" + + request_path = tmp_path / "must-not-be-read.json" + request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(request_path), + "--profile", + "docx-small", + "--iterations", + "1001", + "--reference-hardware", + "pytest-reference", + ], + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "iterations must be between 1 and 100" in completed.stderr + assert str(request_path) not in completed.stderr + assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr From 38a287cc44ffa7ad6893c1a9f3cd496f82ae98a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:42:25 -0700 Subject: [PATCH 164/260] test(perf): bind Office evidence to canonical fixtures --- office/tests/test_performance_measurement.py | 172 +++++++++++++------ 1 file changed, 124 insertions(+), 48 deletions(-) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 94c14d7d..15fc5219 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -8,39 +8,94 @@ from pathlib import Path -def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: - """Measure repeated synthetic renders without copying document content into evidence.""" +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) - sentinel = "PRIVATE-BENCHMARK-PAYLOAD-SENTINEL" - request_path = tmp_path / "buyer-private-name.json" - request_path.write_text( - json.dumps( - { - "format": "docx", - "title": "Synthetic benchmark fixture", - "blocks": [ - {"type": "heading", "level": 1, "text": "Synthetic heading"}, - {"type": "paragraph", "text": sentinel}, - ], - } - ), - encoding="utf-8", - ) +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _measure_command(script: Path, request_path: Path, iterations: str) -> list[str]: + return [ + sys.executable, + str(script), + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + iterations, + "--reference-hardware", + "pytest-reference", + ] + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure a lock-verified synthetic fixture without copying its document body into evidence.""" + + request_path = tmp_path / "synthetic-docx-small.json" + request_bytes = _canonical_docx_small_fixture_bytes() + request_path.write_bytes(request_bytes) script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" completed = subprocess.run( - [ - sys.executable, - str(script), - "--input", - str(request_path), - "--profile", - "docx-small", - "--iterations", - "2", - "--reference-hardware", - "pytest-reference", - ], + _measure_command(script, request_path, "2"), check=False, cwd=Path(__file__).resolve().parents[2], capture_output=True, @@ -52,10 +107,12 @@ def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path assert evidence["contractVersion"] == 1 assert evidence["synthetic"] is True assert evidence["operation"] == "office_render" - assert evidence["profile"] == "docx-small" + assert evidence["fixtureId"] == "docx.small" + assert evidence["profile"] == "small" assert evidence["format"] == "docx" assert evidence["iterations"] == 2 assert evidence["referenceHardware"] == "pytest-reference" + assert evidence["fixtureBytes"] == len(request_bytes) assert len(evidence["fixtureSha256"]) == 64 assert len(evidence["sourceSha"]) == 40 assert evidence["runtime"]["implementation"] @@ -67,17 +124,47 @@ def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path assert sample["durationMs"] >= 0 assert isinstance(sample["peakRssBytes"], int) assert sample["peakRssBytes"] > 0 - assert evidence["summary"]["durationMs"]["p50"] >= 0 - assert evidence["summary"]["durationMs"]["p75"] >= 0 - assert evidence["summary"]["durationMs"]["p95"] >= 0 - assert evidence["summary"]["durationMs"]["max"] >= 0 - assert evidence["summary"]["peakRssBytes"]["max"] > 0 + for percentile in ("p50", "p75", "p95", "max"): + assert evidence["summary"]["durationMs"][percentile] >= 0 + assert evidence["summary"]["peakRssBytes"][percentile] > 0 combined_output = completed.stdout + completed.stderr - assert sentinel not in combined_output + assert MULTILINGUAL_PARAGRAPH not in combined_output assert str(request_path) not in combined_output +def test_measure_office_render_rejects_noncanonical_content_without_leaking_it( + tmp_path: Path, +) -> None: + """Do not label arbitrary document content as canonical synthetic benchmark evidence.""" + + sentinel = "PRIVATE-NONCANONICAL-DOCUMENT-SENTINEL" + request_path = tmp_path / "private-customer-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "private customer document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ), + encoding="utf-8", + ) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel not in completed.stderr + assert str(request_path) not in completed.stderr + + def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( tmp_path: Path, ) -> None: @@ -87,18 +174,7 @@ def test_measure_office_render_rejects_unbounded_iteration_counts_without_readin request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" completed = subprocess.run( - [ - sys.executable, - str(script), - "--input", - str(request_path), - "--profile", - "docx-small", - "--iterations", - "1001", - "--reference-hardware", - "pytest-reference", - ], + _measure_command(script, request_path, "1001"), check=False, cwd=Path(__file__).resolve().parents[2], capture_output=True, From a5275f9ac6aa0956628674ba39f9a6e74d1a455f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:43:42 -0700 Subject: [PATCH 165/260] feat(perf): measure canonical Office render latency and RSS --- office/benchmarks/measure_render.py | 305 ++++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 office/benchmarks/measure_render.py diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py new file mode 100644 index 00000000..96a75f89 --- /dev/null +++ b/office/benchmarks/measure_render.py @@ -0,0 +1,305 @@ +"""Produce privacy-safe timing and peak-RSS evidence for canonical Office fixtures.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +OFFICE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = OFFICE_ROOT.parent +LOCK_PATH = REPOSITORY_ROOT / "benchmarks" / "office-fixtures.lock.json" +SOURCE_ROOT = OFFICE_ROOT / "src" +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from inkspan_office.safe_renderer import write_office_document # noqa: E402 + +MAX_ITERATIONS = 100 +MAX_TOKEN_CODE_UNITS = 128 +TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") +GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") +SUPPORTED_FORMATS = {"docx", "xlsx", "pptx"} + + +class BenchmarkContractError(Exception): + """Raised when benchmark evidence cannot be produced safely and truthfully.""" + + +def _metadata_token(value: str, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) > MAX_TOKEN_CODE_UNITS + or not TOKEN_PATTERN.fullmatch(value) + ): + raise BenchmarkContractError(f"{label} must be a bounded metadata token") + return value + + +def _positive_iterations(value: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= MAX_ITERATIONS: + raise BenchmarkContractError("iterations must be between 1 and 100") + return value + + +def _load_fixture_contract(format_name: str, profile: str) -> tuple[int, str]: + try: + lock = json.loads(LOCK_PATH.read_text(encoding="utf-8")) + if lock.get("contractVersion") != 1 or lock.get("synthetic") is not True: + raise BenchmarkContractError("canonical Office fixture lock is invalid") + record = lock["formats"][format_name][profile] + expected_bytes = record["bytes"] + expected_sha256 = record["sha256"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("canonical Office fixture lock is invalid") from exc + if ( + not isinstance(expected_bytes, int) + or isinstance(expected_bytes, bool) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not SHA256_PATTERN.fullmatch(expected_sha256) + ): + raise BenchmarkContractError("canonical Office fixture lock is invalid") + return expected_bytes, expected_sha256 + + +def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256: str) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be opened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks: list[bytes] = [] + remaining = expected_bytes + while remaining: + chunk = os.read(descriptor, min(remaining, 1024 * 1024)) + if not chunk: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + payload = b"".join(chunks) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be read safely") from exc + finally: + os.close(descriptor) + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _source_sha() -> str: + status = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], + check=False, + capture_output=True, + text=True, + ) + if status.returncode != 0 or status.stdout: + raise BenchmarkContractError("benchmark checkout must be clean") + revision = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + sha = revision.stdout.strip() + if revision.returncode != 0 or not GIT_SHA_PATTERN.fullmatch(sha): + raise BenchmarkContractError("benchmark source revision could not be verified") + return sha + + +def _peak_rss_bytes() -> int: + try: + import resource + except ImportError as exc: # pragma: no cover - benchmark CI is POSIX + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") from exc + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if peak <= 0: + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") + if sys.platform == "darwin": + return int(peak) + return int(peak) * 1024 + + +def _child_measure() -> int: + try: + payload_bytes = sys.stdin.buffer.read() + payload = json.loads(payload_bytes.decode("utf-8")) + if not isinstance(payload, dict): + raise BenchmarkContractError("canonical Office fixture must contain an object") + format_name = payload.get("format") + if format_name not in SUPPORTED_FORMATS: + raise BenchmarkContractError("canonical Office fixture format is unsupported") + with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: + output_path = Path(directory) / f"render.{format_name}" + started = time.perf_counter_ns() + write_office_document(payload, output_path) + duration_ms = (time.perf_counter_ns() - started) / 1_000_000 + peak_rss_bytes = _peak_rss_bytes() + print( + json.dumps( + { + "format": format_name, + "durationMs": round(duration_ms, 6), + "peakRssBytes": peak_rss_bytes, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 0 + except Exception: + print("Office benchmark render sample failed.", file=sys.stderr) + return 2 + + +def _run_sample(payload_bytes: bytes) -> dict[str, Any]: + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--child"], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + ) + if completed.returncode != 0: + raise BenchmarkContractError("Office benchmark render sample failed") + try: + sample = json.loads(completed.stdout.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("Office benchmark render sample was invalid") from exc + if not isinstance(sample, dict): + raise BenchmarkContractError("Office benchmark render sample was invalid") + duration = sample.get("durationMs") + peak_rss = sample.get("peakRssBytes") + format_name = sample.get("format") + if ( + format_name not in SUPPORTED_FORMATS + or not isinstance(duration, (int, float)) + or isinstance(duration, bool) + or not math.isfinite(duration) + or duration < 0 + or not isinstance(peak_rss, int) + or isinstance(peak_rss, bool) + or peak_rss <= 0 + ): + raise BenchmarkContractError("Office benchmark render sample was invalid") + return {"format": format_name, "durationMs": duration, "peakRssBytes": peak_rss} + + +def _percentile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _summary(values: list[float], *, integral: bool) -> dict[str, int | float]: + result: dict[str, int | float] = {} + for name, quantile in (("p50", 0.50), ("p75", 0.75), ("p95", 0.95)): + value = _percentile(values, quantile) + result[name] = int(round(value)) if integral else round(value, 6) + maximum = max(values) + result["max"] = int(maximum) if integral else round(maximum, 6) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Measure a canonical synthetic Inkspan Office fixture") + parser.add_argument("--input", required=True, help="canonical generated Office fixture path") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--reference-hardware", required=True) + return parser + + +def _main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + iterations = _positive_iterations(args.iterations) + profile = _metadata_token(args.fixture_profile, "fixture profile") + reference_hardware = _metadata_token(args.reference_hardware, "reference hardware") + expected_bytes, expected_sha256 = _load_fixture_contract(args.format, profile) + source_sha = _source_sha() + payload_bytes = _read_exact_regular_fixture( + Path(args.input), expected_bytes, expected_sha256 + ) + samples = [_run_sample(payload_bytes) for _ in range(iterations)] + observed_formats = {sample["format"] for sample in samples} + if observed_formats != {args.format}: + raise BenchmarkContractError("Office benchmark render format was inconsistent") + duration_values = [float(sample["durationMs"]) for sample in samples] + rss_values = [float(sample["peakRssBytes"]) for sample in samples] + evidence = { + "contractVersion": 1, + "synthetic": True, + "operation": "office_render", + "fixtureId": f"{args.format}.{profile}", + "format": args.format, + "profile": profile, + "iterations": iterations, + "referenceHardware": reference_hardware, + "fixtureBytes": expected_bytes, + "fixtureSha256": expected_sha256, + "sourceSha": source_sha, + "runtime": { + "implementation": platform.python_implementation(), + "python": platform.python_version(), + "platform": sys.platform, + }, + "samples": samples, + "summary": { + "durationMs": _summary(duration_values, integral=False), + "peakRssBytes": _summary(rss_values, integral=True), + }, + } + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark measurement failed.", file=sys.stderr) + return 2 + + +def main() -> int: + """Run the benchmark command or its isolated one-render child process.""" + + if sys.argv[1:] == ["--child"]: + return _child_measure() + return _main() + + +if __name__ == "__main__": + raise SystemExit(main()) From e6598e866a84d6b1139c74a8bd978566863e878d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 11:34:16 -0700 Subject: [PATCH 166/260] test(perf): bound hung Office render sample RED --- office/tests/test_performance_measurement.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 15fc5219..2307d09b 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -3,10 +3,13 @@ from __future__ import annotations import json +import runpy import subprocess import sys from pathlib import Path +import pytest + MULTILINGUAL_PARAGRAPH = ( "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " @@ -185,3 +188,29 @@ def test_measure_office_render_rejects_unbounded_iteration_counts_without_readin assert "iterations must be between 1 and 100" in completed.stderr assert str(request_path) not in completed.stderr assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr + + +def test_office_render_sample_times_out_without_leaking_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bound a hung renderer child and normalize timeout failure without exposing document data.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_test") + benchmark_error = namespace["BenchmarkContractError"] + run_sample = namespace["_run_sample"] + observed_timeout: list[float | None] = [] + sentinel = b'PRIVATE-HUNG-RENDER-SENTINEL' + + def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + timeout = kwargs.get("timeout") + observed_timeout.append(timeout if isinstance(timeout, (int, float)) else None) + raise subprocess.TimeoutExpired(args[0] if args else "renderer", timeout or 0) + + monkeypatch.setattr(subprocess, "run", _timeout) + + with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: + run_sample(sentinel) + + assert observed_timeout == [120] + assert sentinel.decode() not in str(exc_info.value) From 7128b6c1e0c130b26993466bddd18fac5e2ebc60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 11:36:01 -0700 Subject: [PATCH 167/260] fix(perf): bound Office render sample duration --- office/benchmarks/measure_render.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 96a75f89..9b092eae 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -28,6 +28,7 @@ MAX_ITERATIONS = 100 MAX_TOKEN_CODE_UNITS = 128 +SAMPLE_TIMEOUT_SECONDS = 120 TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") @@ -178,13 +179,17 @@ def _child_measure() -> int: def _run_sample(payload_bytes: bytes) -> dict[str, Any]: - completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), "--child"], - input=payload_bytes, - check=False, - cwd=REPOSITORY_ROOT, - capture_output=True, - ) + try: + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--child"], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + timeout=SAMPLE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise BenchmarkContractError("Office benchmark render sample timed out") from None if completed.returncode != 0: raise BenchmarkContractError("Office benchmark render sample failed") try: From 1572af1c985a87452785c865d0a1c405d0631990 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:26:19 -0700 Subject: [PATCH 168/260] test(perf): require child fixture re-verification --- office/tests/test_performance_measurement.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 2307d09b..c0b21277 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -214,3 +214,36 @@ def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[byt assert observed_timeout == [120] assert sentinel.decode() not in str(exc_info.value) + + +def test_office_render_child_rejects_unverified_private_payload() -> None: + """Require the isolated child to re-verify canonical fixture identity before rendering.""" + + sentinel = "PRIVATE-DIRECT-CHILD-DOCUMENT-SENTINEL" + payload = json.dumps( + { + "format": "docx", + "title": "private direct child document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ).encode() + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--child", + "--format", + "docx", + "--fixture-profile", + "small", + ], + input=payload, + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + ) + + assert completed.returncode != 0 + assert b"input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel.encode() not in completed.stderr From 28053fd4e00a8c7dc439e30e2e203d4be208ce98 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:30:07 -0700 Subject: [PATCH 169/260] fix(perf): bind Office child to canonical fixture --- office/benchmarks/measure_render.py | 56 ++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 9b092eae..4789d264 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -112,6 +112,13 @@ def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256 return payload +def _read_exact_stdin_fixture(expected_bytes: int, expected_sha256: str) -> bytes: + payload = sys.stdin.buffer.read(expected_bytes + 1) + if len(payload) != expected_bytes or hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + def _source_sha() -> str: status = subprocess.run( ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], @@ -146,15 +153,16 @@ def _peak_rss_bytes() -> int: return int(peak) * 1024 -def _child_measure() -> int: +def _child_measure(format_name: str, profile: str) -> int: try: - payload_bytes = sys.stdin.buffer.read() + profile = _metadata_token(profile, "fixture profile") + expected_bytes, expected_sha256 = _load_fixture_contract(format_name, profile) + payload_bytes = _read_exact_stdin_fixture(expected_bytes, expected_sha256) payload = json.loads(payload_bytes.decode("utf-8")) if not isinstance(payload, dict): raise BenchmarkContractError("canonical Office fixture must contain an object") - format_name = payload.get("format") - if format_name not in SUPPORTED_FORMATS: - raise BenchmarkContractError("canonical Office fixture format is unsupported") + if payload.get("format") != format_name: + raise BenchmarkContractError("canonical Office fixture format is inconsistent") with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: output_path = Path(directory) / f"render.{format_name}" started = time.perf_counter_ns() @@ -173,15 +181,26 @@ def _child_measure() -> int: ) ) return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 except Exception: print("Office benchmark render sample failed.", file=sys.stderr) return 2 -def _run_sample(payload_bytes: bytes) -> dict[str, Any]: +def _run_sample(payload_bytes: bytes, format_name: str, profile: str) -> dict[str, Any]: try: completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), "--child"], + [ + sys.executable, + str(Path(__file__).resolve()), + "--child", + "--format", + format_name, + "--fixture-profile", + profile, + ], input=payload_bytes, check=False, cwd=REPOSITORY_ROOT, @@ -200,9 +219,9 @@ def _run_sample(payload_bytes: bytes) -> dict[str, Any]: raise BenchmarkContractError("Office benchmark render sample was invalid") duration = sample.get("durationMs") peak_rss = sample.get("peakRssBytes") - format_name = sample.get("format") + observed_format = sample.get("format") if ( - format_name not in SUPPORTED_FORMATS + observed_format != format_name or not isinstance(duration, (int, float)) or isinstance(duration, bool) or not math.isfinite(duration) @@ -212,7 +231,7 @@ def _run_sample(payload_bytes: bytes) -> dict[str, Any]: or peak_rss <= 0 ): raise BenchmarkContractError("Office benchmark render sample was invalid") - return {"format": format_name, "durationMs": duration, "peakRssBytes": peak_rss} + return {"format": observed_format, "durationMs": duration, "peakRssBytes": peak_rss} def _percentile(values: list[float], quantile: float) -> float: @@ -248,6 +267,13 @@ def _parser() -> argparse.ArgumentParser: return parser +def _child_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Render one verified synthetic Inkspan Office fixture") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + return parser + + def _main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) try: @@ -259,10 +285,7 @@ def _main(argv: list[str] | None = None) -> int: payload_bytes = _read_exact_regular_fixture( Path(args.input), expected_bytes, expected_sha256 ) - samples = [_run_sample(payload_bytes) for _ in range(iterations)] - observed_formats = {sample["format"] for sample in samples} - if observed_formats != {args.format}: - raise BenchmarkContractError("Office benchmark render format was inconsistent") + samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] duration_values = [float(sample["durationMs"]) for sample in samples] rss_values = [float(sample["peakRssBytes"]) for sample in samples] evidence = { @@ -301,8 +324,9 @@ def _main(argv: list[str] | None = None) -> int: def main() -> int: """Run the benchmark command or its isolated one-render child process.""" - if sys.argv[1:] == ["--child"]: - return _child_measure() + if sys.argv[1:2] == ["--child"]: + args = _child_parser().parse_args(sys.argv[2:]) + return _child_measure(args.format, args.fixture_profile) return _main() From a6ab00f3be35ded4db6707dd6c8e8d15ea73f642 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:30:51 -0700 Subject: [PATCH 170/260] test(perf): pass canonical child identity --- office/tests/test_performance_measurement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index c0b21277..b9e6c88f 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -210,7 +210,7 @@ def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[byt monkeypatch.setattr(subprocess, "run", _timeout) with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: - run_sample(sentinel) + run_sample(sentinel, "docx", "small") assert observed_timeout == [120] assert sentinel.decode() not in str(exc_info.value) From 2251be8a734799527f8dce87fcb5a9cc291a5b9d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:08:02 -0700 Subject: [PATCH 171/260] test(perf): bind packed benchmark to verified tarball bytes --- ...ormancePackedArtifactPathStability.test.ts | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 src/performancePackedArtifactPathStability.test.ts diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts new file mode 100644 index 00000000..659d9ed7 --- /dev/null +++ b/src/performancePackedArtifactPathStability.test.ts @@ -0,0 +1,201 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'d'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture( + directory: string, + name: string, + markdownMarker: string, +): { + markdownModuleSha256: string; + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, `${name}-package-source`); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, `${name}-packed`); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + const markdownModule = `export function markdownToHtml(source) { return \`

${source}

\`; }\n`; + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + markdownModule, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'e'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + markdownModuleSha256: sha256(markdownModule), + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function createTarInterpositionShim( + directory: string, + originalTarballPath: string, + adversarialTarballPath: string, +): { environment: NodeJS.ProcessEnv; originalBackupPath: string } { + const shimDirectory = join(directory, 'shim'); + mkdirSync(shimDirectory, { recursive: true }); + const originalBackupPath = join(directory, 'original-package-backup.tgz'); + copyFileSync(originalTarballPath, originalBackupPath); + const realTar = execFileSync('sh', ['-c', 'command -v tar'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + const shimPath = join(shimDirectory, 'tar'); + writeFileSync( + shimPath, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args.at(-1);\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + 'utf8', + ); + chmodSync(shimPath, 0o755); + return { + originalBackupPath, + environment: { + ...process.env, + PATH: `${shimDirectory}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_TEST_REAL_TAR: realTar, + INKSPAN_TEST_ORIGINAL_TARBALL: originalTarballPath, + INKSPAN_TEST_ADVERSARIAL_TARBALL: adversarialTarballPath, + INKSPAN_TEST_ORIGINAL_BACKUP: originalBackupPath, + }, + }; +} + +describe('packed artifact benchmark path stability', () => { + it('measures the same tarball bytes whose package digest was verified', () => { + if (process.platform === 'win32') return; + + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-path-stability-')); + temporaryDirectories.push(directory); + const original = createPackedBenchmarkFixture(directory, 'original', 'verified'); + const adversarial = createPackedBenchmarkFixture(directory, 'adversarial', 'interposed'); + const { environment } = createTarInterpositionShim( + directory, + original.tarballPath, + adversarial.tarballPath, + ); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const outputDirectory = join(directory, 'evidence'); + writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Stable packed artifact"}\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + original.tarballPath, + '--package-sha256', + original.packageSha256, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--runtime-id', + activeRuntimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + const markdownEvidence = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { artifactSha256: string }; + expect(markdownEvidence.artifactSha256).toBe(original.markdownModuleSha256); + }); +}); From 0055f240dc86763008c59228737519cf3a4b55d0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:09:48 -0700 Subject: [PATCH 172/260] test(perf): exercise packed tarball interposition --- src/performancePackedArtifactPathStability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 659d9ed7..ae659c42 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -117,7 +117,7 @@ function createTarInterpositionShim( const shimPath = join(shimDirectory, 'tar'); writeFileSync( shimPath, - `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args.at(-1);\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args[1];\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, 'utf8', ); chmodSync(shimPath, 0o755); From 4033e2c73a4d68fb26c37c6932bd69c156ce2401 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:10:20 -0700 Subject: [PATCH 173/260] fix(perf): snapshot verified packed artifact before extraction --- benchmarks/run-current-suite.mjs | 162 +++++++++++++++++++++++++++++-- 1 file changed, 152 insertions(+), 10 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index cc87a585..030f33e1 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,5 +1,17 @@ import { spawnSync } from 'node:child_process'; -import { dirname, resolve } from 'node:path'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; @@ -7,21 +19,151 @@ import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packageTarballValueIndex = + packedFlags.indexOf('--package-tarball') * 2 + 1; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function readPackedTarballSnapshot(path) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if (metadata.size > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_PACKAGE_BYTES) { + const remainingBudget = MAX_PACKAGE_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function snapshotPackedArguments(argv) { + if (!matchesArguments(argv, packedFlags)) { + return Object.freeze({ argv, temporaryDirectory: null }); + } + + const packageTarballPath = resolve(argv[packageTarballValueIndex]); + const packageBytes = readPackedTarballSnapshot(packageTarballPath); + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-suite-snapshot-'), + ); + const snapshotPath = join(temporaryDirectory, 'package.tgz'); + try { + writeFileSync(snapshotPath, packageBytes, { mode: 0o600 }); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package tarball snapshot could not be prepared.'); + } + + const snapshottedArguments = [...argv]; + snapshottedArguments[packageTarballValueIndex] = snapshotPath; + return Object.freeze({ + argv: snapshottedArguments, + temporaryDirectory, + }); +} function main(argv) { assertCleanSourceCheckout(repositoryRoot); + const snapshotted = snapshotPackedArguments(argv); - const result = spawnSync(process.execPath, [coreRunnerPath, ...argv], { - cwd: repositoryRoot, - stdio: 'inherit', - timeout: 600_000, - }); + try { + const result = spawnSync( + process.execPath, + [coreRunnerPath, ...snapshotted.argv], + { + cwd: repositoryRoot, + stdio: 'inherit', + timeout: 600_000, + }, + ); - if (result.error !== undefined || result.signal !== null) { - throw new Error('Benchmark suite internal runner could not complete.'); - } + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } - process.exitCode = result.status ?? 1; + process.exitCode = result.status ?? 1; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } } try { From c455ec147e0a3d11106a1d4e971bb36d084c5b69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:12:07 -0700 Subject: [PATCH 174/260] fix(test): preserve generated benchmark source interpolation --- src/performancePackedArtifactPathStability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index ae659c42..7ff8c0df 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -50,7 +50,7 @@ function createPackedBenchmarkFixture( mkdirSync(distDirectory, { recursive: true }); mkdirSync(packDirectory, { recursive: true }); - const markdownModule = `export function markdownToHtml(source) { return \`

${source}

\`; }\n`; + const markdownModule = `export function markdownToHtml(source) { return \`

\${source}

\`; }\n`; writeFileSync( join(packageDirectory, 'package.json'), `${JSON.stringify( From 5abc23761f6285b796444f6636a37cd8a9a60238 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:19:26 -0700 Subject: [PATCH 175/260] test(perf): reject mutable existing suite evidence targets --- ...rmanceSuiteExistingOutputAtomicity.test.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/performanceSuiteExistingOutputAtomicity.test.ts diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts new file mode 100644 index 00000000..a1f6cdfb --- /dev/null +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -0,0 +1,104 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('benchmark suite existing-output atomicity', () => { + it('rejects an existing evidence directory before mutating prior evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); + temporaryDirectories.push(directory); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); + }); +}); From 962113322857faa596ff16adc0f618cdb43e1049 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:24:25 -0700 Subject: [PATCH 176/260] fix(perf): preserve existing benchmark evidence directories --- benchmarks/run-current-suite.mjs | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 030f33e1..21b21403 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -22,6 +22,20 @@ const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs') const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); const packedFlags = Object.freeze([ '--input', '--revision-input', @@ -45,6 +59,28 @@ function matchesArguments(argv, expectedFlags) { ); } +function matchingFlags(argv) { + if (matchesArguments(argv, packedFlags)) return packedFlags; + if (matchesArguments(argv, legacyFlags)) return legacyFlags; + return null; +} + +function assertFreshOutputDirectory(argv) { + const flags = matchingFlags(argv); + if (flags === null) return; + const outputValueIndex = flags.indexOf('--output') * 2 + 1; + const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); + let metadata; + try { + metadata = lstatSync(outputDirectory, { throwIfNoEntry: false }); + } catch { + return; + } + if (metadata?.isDirectory()) { + throw new Error('Benchmark suite output directory must not already exist.'); + } +} + function readPackedTarballSnapshot(path) { let pathMetadata; try { @@ -138,6 +174,7 @@ function snapshotPackedArguments(argv) { function main(argv) { assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); const snapshotted = snapshotPackedArguments(argv); try { From 1949f18581bc76551ef73d08202342e37329829f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:06:56 -0700 Subject: [PATCH 177/260] test(perf): cover core existing-output race boundary --- ...rmanceSuiteExistingOutputAtomicity.test.ts | 177 +++++++++++++----- 1 file changed, 126 insertions(+), 51 deletions(-) diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts index a1f6cdfb..063b0ce4 100644 --- a/src/performanceSuiteExistingOutputAtomicity.test.ts +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -15,6 +15,10 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const coreSuitePath = resolve( + repositoryRoot, + 'benchmarks/run-current-suite-core.mjs', +); const temporaryDirectories: string[] = []; afterEach(() => { @@ -27,61 +31,136 @@ function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); } +function existingOutputArguments({ + suite, + markdownInputPath, + markdownModulePath, + revisionInputPath, + revisionModulePath, + markdownModuleSource, + revisionModuleSource, + outputDirectory, +}: { + suite: string; + markdownInputPath: string; + markdownModulePath: string; + revisionInputPath: string; + revisionModulePath: string; + markdownModuleSource: string; + revisionModuleSource: string; + outputDirectory: string; +}) { + return [ + suite, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function makeExistingOutputFixture(directory: string) { + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + return { + markdownInputPath, + revisionInputPath, + markdownModulePath, + revisionModulePath, + outputDirectory, + priorEvidencePath, + markdownModuleSource, + revisionModuleSource, + }; +} + +function expectExistingEvidenceUntouched({ + outputDirectory, + priorEvidencePath, +}: { + outputDirectory: string; + priorEvidencePath: string; +}) { + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); +} + describe('benchmark suite existing-output atomicity', () => { it('rejects an existing evidence directory before mutating prior evidence', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); temporaryDirectories.push(directory); - const markdownInputPath = join(directory, 'input.md'); - const revisionInputPath = join(directory, 'document-envelope.json'); - const markdownModulePath = join(directory, 'markdown.mjs'); - const revisionModulePath = join(directory, 'revision.mjs'); - const outputDirectory = join(directory, 'evidence'); - const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); - const markdownModuleSource = - "export function markdownToHtml(source) { return `

${source}

`; }\n"; - const revisionModuleSource = - "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: suitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); - writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', - 'utf8', + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', ); - writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); - writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); - mkdirSync(outputDirectory); - writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + expectExistingEvidenceUntouched(fixture); + }); + + it('rejects an output directory created after the wrapper preflight', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-core-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); const result = spawnSync( process.execPath, - [ - suitePath, - '--input', - markdownInputPath, - '--module', - markdownModulePath, - '--revision-input', - revisionInputPath, - '--revision-module', - revisionModulePath, - '--profile', - 'small', - '--samples', - '1', - '--source-commit-sha', - 'a'.repeat(40), - '--artifact-sha256', - sha256(markdownModuleSource), - '--revision-artifact-sha256', - sha256(revisionModuleSource), - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + existingOutputArguments({ suite: coreSuitePath, ...fixture }), { cwd: repositoryRoot, encoding: 'utf8', @@ -95,10 +174,6 @@ describe('benchmark suite existing-output atomicity', () => { expect(result.stderr).toBe( 'Benchmark suite output directory must not already exist.\n', ); - expect(readFileSync(priorEvidencePath, 'utf8')).toBe( - '{"status":"accepted"}\n', - ); - expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); - expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); + expectExistingEvidenceUntouched(fixture); }); }); From 92683c28e68ac588327cb21fb851a30bfe260150 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:10:35 -0700 Subject: [PATCH 178/260] fix(perf): fail closed on existing evidence output --- benchmarks/run-current-suite-core.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index d9717fc3..a18479e8 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -58,6 +58,8 @@ const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; const OUTPUT_DIRECTORY_ERROR = 'Benchmark suite output directory must be a non-symlink directory.'; +const OUTPUT_DIRECTORY_EXISTS_ERROR = + 'Benchmark suite output directory must not already exist.'; function matchesArguments(argv, expectedFlags) { return ( @@ -217,7 +219,7 @@ function prepareOutputDirectory(path) { if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } - return false; + throw new Error(OUTPUT_DIRECTORY_EXISTS_ERROR); } try { From c16fe9c471e221f8a8d04df8149404b8b895bb44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:13:32 -0700 Subject: [PATCH 179/260] test(perf): preserve existing measurement evidence --- ...eMeasurementProducerOutputHardlink.test.ts | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts index e3210db3..5aeea7a0 100644 --- a/src/performanceMeasurementProducerOutputHardlink.test.ts +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -52,7 +52,7 @@ function commonArguments( ]; } -function expectHardlinkFailure( +function expectOutputPreservedFailure( script: string, args: string[], sentinel: string, @@ -70,7 +70,7 @@ function expectHardlinkFailure( expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); } -describe('benchmark producer output hard-link safety', () => { +describe('benchmark producer output immutability', () => { it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); const input = join(root, 'document.md'); @@ -85,7 +85,7 @@ describe('benchmark producer output hard-link safety', () => { writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); linkSync(sentinel, output); - expectHardlinkFailure( + expectOutputPreservedFailure( markdownScript, commonArguments(input, module, sha256(moduleSource), output), sentinel, @@ -115,7 +115,7 @@ describe('benchmark producer output hard-link safety', () => { writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); linkSync(sentinel, output); - expectHardlinkFailure( + expectOutputPreservedFailure( revisionScript, commonArguments(input, module, sha256(moduleSource), output), sentinel, @@ -125,4 +125,55 @@ describe('benchmark producer output hard-link safety', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before Markdown measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-existing-output-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Markdown benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-existing-output-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Revision benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 856c1e9932daf84290136db4cb4cc692ce73e864 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:15:41 -0700 Subject: [PATCH 180/260] fix(perf): preserve existing markdown evidence --- benchmarks/measure-markdown.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 68f1cae1..51d437ec 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -30,6 +30,8 @@ const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const OUTPUT_DIRECTORY_ERROR = 'Markdown benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Markdown benchmark output must not already exist.'; function resolveArguments(argv) { const expectedFlags = [ @@ -301,7 +303,7 @@ function writeMeasurementOutput(path, content) { } assertNoSymlinkOutputAncestors(path); try { - writeFileSync(path, content, 'utf8'); + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); } catch { throw new Error('Markdown benchmark output could not be written.'); } @@ -368,6 +370,9 @@ async function main() { if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { throw new Error('Markdown benchmark output must not be multiply linked.'); } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 1b7710a5e8bf3300182b6f5ec4bc05be840c57f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:16:57 -0700 Subject: [PATCH 181/260] fix(perf): preserve existing revision evidence --- benchmarks/measure-revision-evidence.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index a209e3e0..5a69b2b6 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -29,6 +29,8 @@ const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const OUTPUT_DIRECTORY_ERROR = 'Revision benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Revision benchmark output must not already exist.'; function resolveArguments(argv) { const expectedFlags = [ @@ -285,7 +287,7 @@ function writeMeasurementOutput(path, content) { } assertNoSymlinkOutputAncestors(path); try { - writeFileSync(path, content, 'utf8'); + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); } catch { throw new Error('Revision benchmark output could not be written.'); } @@ -377,6 +379,9 @@ async function main() { if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { throw new Error('Revision benchmark output must not be multiply linked.'); } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 71b1fc5f7aa528450d6270c7949fcdc95e313af7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:09:16 -0700 Subject: [PATCH 182/260] test(perf): require HTML serialization measurement --- ...rmanceHtmlSerializationMeasurement.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/performanceHtmlSerializationMeasurement.test.ts diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts new file mode 100644 index 00000000..e7d016c2 --- /dev/null +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -0,0 +1,95 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('HTML serialization performance measurement', () => { + it('measures packed htmlToMarkdown without falling back to markdownToHtml', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-html-serialization-')); + const input = join(root, 'small.html'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(input, '

Synthetic benchmark fixture

\n', 'utf8'); + writeFileSync( + modulePath, + [ + "export function markdownToHtml() { throw new Error('wrong serialization direction'); }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const evidence = JSON.parse(readFileSync(output, 'utf8')) as { + benchmarkId: string; + unit: string; + documentProfile: string; + samples: unknown[]; + }; + expect(evidence.benchmarkId).toBe('html-serialization-small'); + expect(evidence.unit).toBe('ms'); + expect(evidence.documentProfile).toBe('small'); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 8851d5dd0e9fbd62bf365fb7f23d86aafe5c0560 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:12:23 -0700 Subject: [PATCH 183/260] feat(perf): measure HTML serialization direction --- benchmarks/measure-markdown.mjs | 117 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 30 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 51d437ec..424e5a22 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -22,6 +22,10 @@ const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SERIALIZATION_OPERATIONS = new Set([ + 'markdown-to-html', + 'html-to-markdown', +]); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -32,32 +36,61 @@ const OUTPUT_DIRECTORY_ERROR = 'Markdown benchmark output directory must be a non-symlink directory.'; const OUTPUT_EXISTS_ERROR = 'Markdown benchmark output must not already exist.'; +const LEGACY_FLAGS = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const OPERATION_FLAGS = Object.freeze([ + '--input', + '--module', + '--operation', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} function resolveArguments(argv) { - const expectedFlags = [ - '--input', - '--module', - '--profile', - '--samples', - '--source-commit-sha', - '--artifact-sha256', - '--runtime-id', - '--reference-hardware-id', - '--output', - ]; - if ( - argv.length !== expectedFlags.length * 2 || - expectedFlags.some((flag, index) => argv[index * 2] !== flag) || - expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) - ) { + let values; + let operation = 'markdown-to-html'; + if (matchesArguments(argv, OPERATION_FLAGS)) { + values = valuesForArguments(argv, OPERATION_FLAGS); + operation = values['--operation']; + if (!SERIALIZATION_OPERATIONS.has(operation)) { + throw new Error('Markdown benchmark serialization operation is invalid.'); + } + } else if (matchesArguments(argv, LEGACY_FLAGS)) { + values = valuesForArguments(argv, LEGACY_FLAGS); + } else { throw new Error( 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } - const values = Object.fromEntries( - expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), - ); const profile = values['--profile']; if (!DOCUMENT_PROFILES.has(profile)) { throw new Error('Markdown benchmark profile is invalid.'); @@ -94,6 +127,7 @@ function resolveArguments(argv) { return Object.freeze({ inputPath: resolve(values['--input']), modulePath: values['--module'], + operation, profile, sampleCount, sourceCommitSha, @@ -286,11 +320,28 @@ async function loadMeasuredModule(modulePath) { } } -function runMeasuredMarkdownToHtml(markdownToHtml, source) { +function serializationContract(operation) { + if (operation === 'html-to-markdown') { + return Object.freeze({ + exportName: 'htmlToMarkdown', + benchmarkPrefix: 'html-serialization', + executionFailure: 'Measured htmlToMarkdown() execution failed.', + returnFailure: 'Measured htmlToMarkdown() must return a string.', + }); + } + return Object.freeze({ + exportName: 'markdownToHtml', + benchmarkPrefix: 'markdown-serialization', + executionFailure: 'Measured markdownToHtml() execution failed.', + returnFailure: 'Measured markdownToHtml() must return a string.', + }); +} + +function runMeasuredSerialization(serializer, source, failureMessage) { try { - return markdownToHtml(source); + return serializer(source); } catch { - throw new Error('Measured markdownToHtml() execution failed.'); + throw new Error(failureMessage); } } @@ -331,24 +382,30 @@ async function main() { verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const measuredModule = await loadMeasuredModule(modulePath); - if (typeof measuredModule.markdownToHtml !== 'function') { - throw new Error('Measured Markdown module must export markdownToHtml().'); + const contract = serializationContract(args.operation); + const serializer = measuredModule[contract.exportName]; + if (typeof serializer !== 'function') { + throw new Error( + `Measured Markdown module must export ${contract.exportName}().`, + ); } - const warmup = runMeasuredMarkdownToHtml( - measuredModule.markdownToHtml, + const warmup = runMeasuredSerialization( + serializer, source, + contract.executionFailure, ); if (typeof warmup !== 'string') { - throw new Error('Measured markdownToHtml() must return a string.'); + throw new Error(contract.returnFailure); } const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - const output = runMeasuredMarkdownToHtml( - measuredModule.markdownToHtml, + const output = runMeasuredSerialization( + serializer, source, + contract.executionFailure, ); const elapsed = performance.now() - start; if ( @@ -378,7 +435,7 @@ async function main() { `${JSON.stringify( { contractVersion: 1, - benchmarkId: `markdown-serialization-${args.profile}`, + benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, From cf8fd2e3dd917c1874781cfb1000dcc1d0f71c1e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:19:58 -0700 Subject: [PATCH 184/260] test(perf): require HTML serialization in benchmark suite --- ...anceHtmlSerializationSuiteContract.test.ts | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 src/performanceHtmlSerializationSuiteContract.test.ts diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts new file mode 100644 index 00000000..4a401681 --- /dev/null +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -0,0 +1,140 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('single-command HTML serialization benchmark contract', () => { + it('measures HTML-to-Markdown serialization alongside Markdown and revision evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-html-suite-')); + const markdownInput = join(directory, 'document.md'); + const htmlInput = join(directory, 'document.html'); + const markdownModule = join(directory, 'markdown.mjs'); + const revisionInput = join(directory, 'document-envelope.json'); + const revisionModule = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const markdownModuleSource = [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + try { + writeFileSync(markdownInput, '# Buyer benchmark\n', 'utf8'); + writeFileSync(htmlInput, '

Buyer benchmark

\n', 'utf8'); + writeFileSync(markdownModule, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInput, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModule, revisionModuleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInput, + '--html-input', + htmlInput, + '--module', + markdownModule, + '--revision-input', + revisionInput, + '--revision-module', + revisionModule, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const manifest = JSON.parse(result.stdout.trim()) as Record; + expect(manifest).toMatchObject({ + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + status: 'completed', + }); + + const samples = JSON.parse( + readFileSync( + join(outputDirectory, 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { + benchmarkId?: unknown; + documentProfile?: unknown; + samples?: unknown[]; + }; + expect(samples.benchmarkId).toBe('html-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.json', + ), + 'utf8', + ), + ) as { benchmarkId?: unknown }; + expect(summary.benchmarkId).toBe('html-serialization-small'); + expect( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.txt', + ), + 'utf8', + ), + ).toContain('html-serialization-small'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); From 94ad4f81791117aaf3ffd70dded27c5006f1101b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:23:56 -0700 Subject: [PATCH 185/260] feat(perf): include HTML serialization in suite --- benchmarks/run-current-suite.mjs | 157 ++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 3 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 21b21403..0eff1312 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -19,7 +19,13 @@ import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const markdownMeasurementPath = resolve( + benchmarkDirectory, + 'measure-markdown.mjs', +); +const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); const legacyFlags = Object.freeze([ @@ -36,6 +42,21 @@ const legacyFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const htmlLegacyFlags = Object.freeze([ + '--input', + '--html-input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); const packedFlags = Object.freeze([ '--input', '--revision-input', @@ -59,7 +80,18 @@ function matchesArguments(argv, expectedFlags) { ); } +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function argumentsForFlags(values, flags) { + return flags.flatMap((flag) => [flag, values[flag]]); +} + function matchingFlags(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; if (matchesArguments(argv, packedFlags)) return packedFlags; if (matchesArguments(argv, legacyFlags)) return legacyFlags; return null; @@ -172,9 +204,118 @@ function snapshotPackedArguments(argv) { }); } -function main(argv) { - assertCleanSourceCheckout(repositoryRoot); - assertFreshOutputDirectory(argv); +function runBoundedNode(scriptPath, args, failureMessage) { + const result = spawnSync(process.execPath, [scriptPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function parseCoreManifest(stdout) { + let manifest; + try { + manifest = JSON.parse(stdout.trim()); + } catch { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.status !== 'completed' + ) { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + return manifest; +} + +function runHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, htmlLegacyFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const legacyArguments = argumentsForFlags(values, legacyFlags); + let coreCompleted = false; + + try { + const coreStdout = runBoundedNode( + coreRunnerPath, + legacyArguments, + 'Benchmark suite internal runner failed.', + ); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + '--input', + values['--html-input'], + '--module', + values['--module'], + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + values['--artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runExistingSuite(argv) { const snapshotted = snapshotPackedArguments(argv); try { @@ -203,6 +344,16 @@ function main(argv) { } } +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); + if (matchesArguments(argv, htmlLegacyFlags)) { + runHtmlSerializationSuite(argv); + return; + } + runExistingSuite(argv); +} + try { main(process.argv.slice(2)); } catch (error) { From 0d8557daf4b5be8ced592312fe95c4fbecafba0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:28:46 -0700 Subject: [PATCH 186/260] test(perf): require packed HTML serialization evidence --- ...ormancePackedArtifactSuiteContract.test.ts | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 35098ef8..5e3eb674 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -59,7 +59,11 @@ function createPackedBenchmarkFixture(directory: string): { ); writeFileSync( join(distDirectory, 'cwl-markdown.js'), - "export function markdownToHtml(source) { return `

${source}

`; }\n", + [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), 'utf8', ); writeFileSync( @@ -100,8 +104,10 @@ function packedSuiteArguments(options: { tarballPath: string; }): string[] { const markdownInputPath = join(options.directory, 'input.md'); + const htmlInputPath = join(options.directory, 'input.html'); const revisionInputPath = join(options.directory, 'document-envelope.json'); writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); writeFileSync( revisionInputPath, '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', @@ -111,6 +117,8 @@ function packedSuiteArguments(options: { suitePath, '--input', markdownInputPath, + '--html-input', + htmlInputPath, '--revision-input', revisionInputPath, '--package-tarball', @@ -166,8 +174,22 @@ describe('packed artifact benchmark suite contract', () => { packageName: '@contextualwisdomlab/cwl-editor', packageVersion: '0.0.0-benchmark-fixture', packageSha256: packed.packageSha256, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', status: 'completed', }); + + const htmlSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); + expect(htmlSamples.samples).toHaveLength(2); }); it('rejects a runtime identifier that does not match the active Node process', () => { From c220538e5126bcd764ec4f47a4fbb94df414e050 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:39:14 -0700 Subject: [PATCH 187/260] test(perf): reject mismatched benchmark source checkout --- ...ceSourceCheckoutProvenanceContract.test.ts | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts index 94f37cc2..34ab09d0 100644 --- a/src/performanceSourceCheckoutProvenanceContract.test.ts +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -15,7 +15,7 @@ const temporaryDirectories: string[] = []; const probe = ` import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; try { - assertCleanSourceCheckout(process.argv[1]); + assertCleanSourceCheckout(process.argv[1], process.argv[2]); process.stdout.write('clean\\n'); } catch (error) { process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); @@ -45,10 +45,17 @@ function createRepository(): string { return directory; } -function probeCheckout(directory: string) { +function headSha(directory: string): string { + return execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: directory, + encoding: 'utf8', + }).trim(); +} + +function probeCheckout(directory: string, expectedCommitSha = headSha(directory)) { return spawnSync( process.execPath, - ['--input-type=module', '--eval', probe, directory], + ['--input-type=module', '--eval', probe, directory, expectedCommitSha], { cwd: repositoryRoot, encoding: 'utf8', @@ -60,7 +67,7 @@ function probeCheckout(directory: string) { } describe('benchmark source checkout provenance', () => { - it('accepts a clean source checkout', () => { + it('accepts a clean source checkout at the claimed source commit', () => { const directory = createRepository(); const result = probeCheckout(directory); @@ -71,6 +78,23 @@ describe('benchmark source checkout provenance', () => { expect(result.stderr).toBe(''); }); + it('rejects a clean checkout when the claimed source commit is not HEAD', () => { + const directory = createRepository(); + const actualHead = headSha(directory); + const mismatchedCommit = actualHead === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + const result = probeCheckout(directory, mismatchedCommit); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(actualHead); + expect(result.stderr).not.toContain(mismatchedCommit); + }); + it('rejects untracked source state without disclosing paths', () => { const directory = createRepository(); const untrackedPath = join(directory, 'untracked-secret-name.txt'); From 67b499c96301ae9df7b531623fee20e3caa8a53f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:39:52 -0700 Subject: [PATCH 188/260] fix(perf): bind benchmark evidence to checked-out source --- benchmarks/source-checkout-provenance.mjs | 34 ++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs index e2362381..c1cfdf79 100644 --- a/benchmarks/source-checkout-provenance.mjs +++ b/benchmarks/source-checkout-provenance.mjs @@ -2,7 +2,30 @@ import { spawnSync } from 'node:child_process'; const MAX_STATUS_BYTES = 1024 * 1024; -export function assertCleanSourceCheckout(repositoryRoot) { +function checkedOutHeadSha(repositoryRoot) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout identity could not be verified.', + ); + } + + return result.stdout.trim(); +} + +export function assertCleanSourceCheckout(repositoryRoot, expectedSourceCommitSha) { const result = spawnSync( 'git', ['status', '--porcelain=v1', '--untracked-files=all'], @@ -31,4 +54,13 @@ export function assertCleanSourceCheckout(repositoryRoot) { 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', ); } + + if ( + expectedSourceCommitSha !== undefined && + checkedOutHeadSha(repositoryRoot) !== expectedSourceCommitSha + ) { + throw new Error( + 'Benchmark suite source commit does not match checked-out HEAD.', + ); + } } From fd73da75c76dd23de82f5a870200abbae1e8d62f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:43:37 -0700 Subject: [PATCH 189/260] test(perf): bind suite source provenance to checkout --- ...formanceSingleCommandSuiteContract.test.ts | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 0eae1cc5..4c0b0dac 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -17,6 +17,15 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -32,6 +41,7 @@ function benchmarkArguments( revisionModulePath: string, revisionArtifactSha256: string, outputDirectory: string, + sourceCommitSha = currentSourceCommitSha, ): string[] { return [ suitePath, @@ -48,7 +58,7 @@ function benchmarkArguments( '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + sourceCommitSha, '--artifact-sha256', markdownArtifactSha256, '--revision-artifact-sha256', @@ -131,7 +141,7 @@ describe('single-command benchmark suite contract', () => { contractVersion: 1, documentProfile: 'small', sampleCount: 2, - sourceCommitSha: 'a'.repeat(40), + sourceCommitSha: currentSourceCommitSha, runtimeId: 'node-22.0.0', referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', @@ -188,6 +198,44 @@ describe('single-command benchmark suite contract', () => { ).toContain('revision-evidence-small'); }); + it('rejects a claimed source commit that is not the checked-out HEAD', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-source-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const mismatchedCommit = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + mismatchedCommit, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(currentSourceCommitSha); + expect(result.stderr).not.toContain(mismatchedCommit); + expect(existsSync(outputDirectory)).toBe(false); + }); + it('removes partial suite evidence when a downstream measurement fails', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); From 5547e4d1d35c21f5f9150f96ef660688d9eb7104 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:43:59 -0700 Subject: [PATCH 190/260] test(perf): use live checkout SHA for HTML suite --- ...erformanceHtmlSerializationSuiteContract.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index 4a401681..e4a6eadb 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, @@ -12,6 +12,15 @@ import { describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); @@ -63,7 +72,7 @@ describe('single-command HTML serialization benchmark contract', () => { '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + currentSourceCommitSha, '--artifact-sha256', sha256(markdownModuleSource), '--revision-artifact-sha256', From aa9be79eda40c0811d0e58b4978080bdea641335 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:45:02 -0700 Subject: [PATCH 191/260] fix(perf): reject wrong source SHA before benchmark evidence --- benchmarks/run-current-suite.mjs | 48 +++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 0eff1312..b2e405c2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -97,20 +97,50 @@ function matchingFlags(argv) { return null; } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + function assertFreshOutputDirectory(argv) { const flags = matchingFlags(argv); if (flags === null) return; const outputValueIndex = flags.indexOf('--output') * 2 + 1; const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); - let metadata; - try { - metadata = lstatSync(outputDirectory, { throwIfNoEntry: false }); - } catch { - return; + + let current = outputDirectory; + while (true) { + const metadata = inspectOutputPath(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + if (current === outputDirectory && metadata !== undefined) { + if (!metadata.isDirectory()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + throw new Error('Benchmark suite output directory must not already exist.'); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; } - if (metadata?.isDirectory()) { - throw new Error('Benchmark suite output directory must not already exist.'); +} + +function claimedLegacySourceCommitSha(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) { + return valuesForArguments(argv, htmlLegacyFlags)['--source-commit-sha']; } + if (matchesArguments(argv, legacyFlags)) { + return valuesForArguments(argv, legacyFlags)['--source-commit-sha']; + } + return undefined; } function readPackedTarballSnapshot(path) { @@ -347,6 +377,10 @@ function runExistingSuite(argv) { function main(argv) { assertCleanSourceCheckout(repositoryRoot); assertFreshOutputDirectory(argv); + const expectedLegacySourceCommitSha = claimedLegacySourceCommitSha(argv); + if (expectedLegacySourceCommitSha !== undefined) { + assertCleanSourceCheckout(repositoryRoot, expectedLegacySourceCommitSha); + } if (matchesArguments(argv, htmlLegacyFlags)) { runHtmlSerializationSuite(argv); return; From 0001bb9ea99083c617854fb7e0f4a8f3c744badf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:51:47 -0700 Subject: [PATCH 192/260] fix(perf): restore packed HTML benchmark composition --- benchmarks/run-current-suite.mjs | 215 ++++++++++++++++++++++++++++--- 1 file changed, 194 insertions(+), 21 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index b2e405c2..21ca46e5 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; import { closeSync, @@ -25,9 +26,11 @@ const markdownMeasurementPath = resolve( ); const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const PACKED_MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; const legacyFlags = Object.freeze([ '--input', '--module', @@ -69,8 +72,19 @@ const packedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); -const packageTarballValueIndex = - packedFlags.indexOf('--package-tarball') * 2 + 1; +const packedHtmlFlags = Object.freeze([ + '--input', + '--html-input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); function matchesArguments(argv, expectedFlags) { return ( @@ -92,6 +106,7 @@ function argumentsForFlags(values, flags) { function matchingFlags(argv) { if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; + if (matchesArguments(argv, packedHtmlFlags)) return packedHtmlFlags; if (matchesArguments(argv, packedFlags)) return packedFlags; if (matchesArguments(argv, legacyFlags)) return legacyFlags; return null; @@ -209,10 +224,17 @@ function readPackedTarballSnapshot(path) { } function snapshotPackedArguments(argv) { - if (!matchesArguments(argv, packedFlags)) { + const flags = matchesArguments(argv, packedHtmlFlags) + ? packedHtmlFlags + : matchesArguments(argv, packedFlags) + ? packedFlags + : null; + if (flags === null) { return Object.freeze({ argv, temporaryDirectory: null }); } + const packageTarballValueIndex = + flags.indexOf('--package-tarball') * 2 + 1; const packageTarballPath = resolve(argv[packageTarballValueIndex]); const packageBytes = readPackedTarballSnapshot(packageTarballPath); const temporaryDirectory = mkdtempSync( @@ -252,6 +274,26 @@ function runBoundedNode(scriptPath, args, failureMessage) { return result.stdout; } +function runCoreNodePreservingError(args) { + const result = spawnSync(process.execPath, [coreRunnerPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + if (result.status !== 0) { + const message = result.stderr.trimEnd(); + throw new Error( + message.length > 0 ? message : 'Benchmark suite internal runner failed.', + ); + } + return result.stdout; +} + function parseCoreManifest(stdout) { let manifest; try { @@ -270,6 +312,61 @@ function parseCoreManifest(stdout) { return manifest; } +function readPackedMarkdownModule(tarballPath) { + const result = spawnSync( + 'tar', + ['-xOzf', tarballPath, PACKED_MARKDOWN_MODULE_ENTRY], + { + cwd: repositoryRoot, + maxBuffer: MAX_MODULE_BYTES + 1, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }, + ); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > MAX_MODULE_BYTES + ) { + throw new Error('Benchmark suite packed Markdown module could not be read.'); + } + return result.stdout; +} + +function assertPackedSnapshotDigest(tarballPath, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(readPackedTarballSnapshot(tarballPath)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function htmlSerializationEvidenceArguments(values, modulePath, artifactSha256) { + return [ + '--input', + values['--html-input'], + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + artifactSha256, + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]; +} + function runHtmlSerializationSuite(argv) { const values = valuesForArguments(argv, htmlLegacyFlags); const outputDirectory = resolve(repositoryRoot, values['--output']); @@ -298,24 +395,85 @@ function runHtmlSerializationSuite(argv) { runBoundedNode( markdownMeasurementPath, [ - '--input', - values['--html-input'], - '--module', - values['--module'], - '--operation', - 'html-to-markdown', - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], + ...htmlSerializationEvidenceArguments( + values, + values['--module'], + values['--artifact-sha256'], + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runPackedHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, packedHtmlFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const coreArguments = argumentsForFlags(values, packedFlags); + const snapshotted = snapshotPackedArguments(coreArguments); + let coreCompleted = false; + + try { + const coreStdout = runCoreNodePreservingError(snapshotted.argv); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const snapshotValues = valuesForArguments(snapshotted.argv, packedFlags); + const snapshotTarballPath = snapshotValues['--package-tarball']; + const markdownModuleBytes = readPackedMarkdownModule(snapshotTarballPath); + const markdownModulePath = join( + snapshotted.temporaryDirectory, + 'cwl-markdown.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes, { mode: 0o600 }); + } catch { + throw new Error('Benchmark suite packed Markdown module could not be prepared.'); + } + const markdownArtifactSha256 = createHash('sha256') + .update(markdownModuleBytes) + .digest('hex'); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + markdownModulePath, + markdownArtifactSha256, + ), '--output', samplesPath, ], @@ -326,6 +484,10 @@ function runHtmlSerializationSuite(argv) { ['--input', samplesPath, '--output', summaryDirectory], 'Benchmark suite HTML serialization summary failed.', ); + assertPackedSnapshotDigest( + snapshotTarballPath, + values['--package-sha256'], + ); process.stdout.write( `${JSON.stringify({ @@ -342,6 +504,13 @@ function runHtmlSerializationSuite(argv) { rmSync(outputDirectory, { recursive: true, force: true }); } throw error; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } } } @@ -385,6 +554,10 @@ function main(argv) { runHtmlSerializationSuite(argv); return; } + if (matchesArguments(argv, packedHtmlFlags)) { + runPackedHtmlSerializationSuite(argv); + return; + } runExistingSuite(argv); } From 48363bab55a61bff1fdece63d4322eee457691de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:36:27 -0700 Subject: [PATCH 193/260] test(perf): reject false producer provenance --- ...ntProducerSourceProvenanceContract.test.ts | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/performanceMeasurementProducerSourceProvenanceContract.test.ts diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts new file mode 100644 index 00000000..3d64d83a --- /dev/null +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const mismatchedSourceCommitSha = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); +const activeRuntimeId = `node-${process.versions.node}`; +const mismatchedRuntimeId = + activeRuntimeId === 'node-99.99.99' ? 'node-98.98.98' : 'node-99.99.99'; +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function markdownInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +): { result: ReturnType; output: string } { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Provenance fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +function revisionInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +): { result: ReturnType; output: string } { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Provenance fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +describe('direct benchmark producer source/runtime provenance', () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied source SHA that is not the checked-out HEAD for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-')); + try { + const { result, output } = invoke( + root, + mismatchedSourceCommitSha, + activeRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied runtime ID that is not the active Node runtime for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-runtime-')); + try { + const { result, output } = invoke( + root, + currentSourceCommitSha, + mismatchedRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); From a83a4251e337197446034fb5834c36b4c059966e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:40:20 -0700 Subject: [PATCH 194/260] fix(perf): bind Markdown evidence to live source --- benchmarks/measure-markdown.mjs | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 424e5a22..14904b70 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; import { performance } from 'node:perf_hooks'; import { closeSync, @@ -15,6 +16,8 @@ import { import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -138,6 +141,37 @@ function resolveArguments(argv) { }); } +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { let pathMetadata; try { @@ -380,6 +414,7 @@ async function main() { ); } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); const contract = serializationContract(args.operation); From d4e7883c943e37867dca512efae284646d3dacae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:41:24 -0700 Subject: [PATCH 195/260] fix(perf): bind revision evidence to live source --- benchmarks/measure-revision-evidence.mjs | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 5a69b2b6..782063cb 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; import { performance } from 'node:perf_hooks'; import { closeSync, @@ -15,6 +16,8 @@ import { import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -103,6 +106,37 @@ function resolveArguments(argv) { }); } +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { let pathMetadata; try { @@ -340,6 +374,7 @@ async function main() { ); } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); if ( From 19c8c28abdcf6aedf658a7f047f9a55ca03c1b6e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:44:33 -0700 Subject: [PATCH 196/260] fix(test): preserve producer provenance typing --- ...ormanceMeasurementProducerSourceProvenanceContract.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts index 3d64d83a..865d993b 100644 --- a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -38,7 +38,7 @@ function markdownInvocation( root: string, sourceCommitSha: string, runtimeId: string, -): { result: ReturnType; output: string } { +) { const input = join(root, 'document.md'); const modulePath = join(root, 'markdown.mjs'); const output = join(root, 'markdown-samples.json'); @@ -83,7 +83,7 @@ function revisionInvocation( root: string, sourceCommitSha: string, runtimeId: string, -): { result: ReturnType; output: string } { +) { const input = join(root, 'document-envelope.json'); const modulePath = join(root, 'revision.mjs'); const output = join(root, 'revision-samples.json'); From fa91c06e13210c5a35368c143e7a2049f1df90a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:49:48 -0700 Subject: [PATCH 197/260] fix(test): use live Markdown benchmark provenance --- ...ormanceMarkdownMeasurementContract.test.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 87edd0e3..0ec5fdc2 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -25,14 +25,23 @@ interface BenchmarkSamples { readonly samples: number[]; } +const repositoryRoot = process.cwd(); const measurementScript = resolve( - process.cwd(), + repositoryRoot, 'benchmarks/measure-markdown.mjs', ); -const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); -const RUNTIME_ID = 'node-22.18.0'; +const RUNTIME_ID = `node-${process.versions.node}`; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function fileSha256(path: string): string { @@ -89,7 +98,7 @@ describe('Markdown runtime measurement contract', () => { execFileSync( process.execPath, measurementArguments(input, modulePath, samplesPath, artifactSha256), - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); const samples = JSON.parse( @@ -117,7 +126,7 @@ describe('Markdown runtime measurement contract', () => { execFileSync( process.execPath, [summaryScript, '--input', samplesPath, '--output', summaryDirectory], - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); const summary = JSON.parse( readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), @@ -144,7 +153,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -176,7 +185,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -202,7 +211,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, modulePath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -229,7 +238,7 @@ describe('Markdown runtime measurement contract', () => { 'https://example.invalid/markdown.mjs', samplesPath, ), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); From d46dc2f38754b140841a4dd6b7f0c771dbcd65da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:51:03 -0700 Subject: [PATCH 198/260] fix(test): use live revision benchmark provenance --- ...ormanceRevisionMeasurementContract.test.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index d7258f6a..8df70188 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -23,13 +23,22 @@ interface BenchmarkSamples { readonly samples: number[]; } +const repositoryRoot = process.cwd(); const measurementScript = resolve( - process.cwd(), + repositoryRoot, 'benchmarks/measure-revision-evidence.mjs', ); -const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function fileSha256(path: string): string { @@ -100,7 +109,7 @@ describe('revision-evidence runtime measurement contract', () => { ); execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { - cwd: process.cwd(), + cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -130,7 +139,7 @@ describe('revision-evidence runtime measurement contract', () => { execFileSync( process.execPath, [summaryScript, '--input', samplesPath, '--output', summaryDirectory], - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); expect( JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), @@ -156,7 +165,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -187,7 +196,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -221,7 +230,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -253,7 +262,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); From c3d9fa6437ed11dccd77b051553aabebefc616c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:52:10 -0700 Subject: [PATCH 199/260] fix(test): use active suite runtime provenance --- src/performanceSingleCommandSuiteContract.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 4c0b0dac..ca04e242 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -26,6 +26,7 @@ const currentSourceCommitSha = execFileSync( stdio: ['ignore', 'pipe', 'pipe'], }, ).trim(); +const currentRuntimeId = `node-${process.versions.node}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -64,7 +65,7 @@ function benchmarkArguments( '--revision-artifact-sha256', revisionArtifactSha256, '--runtime-id', - 'node-22.0.0', + currentRuntimeId, '--reference-hardware-id', `refhw-sha256-${'b'.repeat(64)}`, '--output', @@ -142,7 +143,7 @@ describe('single-command benchmark suite contract', () => { documentProfile: 'small', sampleCount: 2, sourceCommitSha: currentSourceCommitSha, - runtimeId: 'node-22.0.0', + runtimeId: currentRuntimeId, referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', From 65cb3023dfd6548900ae6d56c492dc769c01dd6c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:07:23 -0700 Subject: [PATCH 200/260] fix(test): bind producer immutability fixtures to active provenance --- ...erformanceMeasurementProducerOutputHardlink.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts index 5aeea7a0..ffc4d283 100644 --- a/src/performanceMeasurementProducerOutputHardlink.test.ts +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { linkSync, mkdtempSync, @@ -16,8 +16,12 @@ const revisionScript = resolve( process.cwd(), 'benchmarks/measure-revision-evidence.mjs', ); -const sourceCommitSha = 'a'.repeat(40); -const runtimeId = 'node-22.18.0'; +const sourceCommitSha = execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const runtimeId = `node-${process.versions.node}`; const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; function sha256(content: string): string { From 8f1480adf16cfb41d6274dc57c94a3bd2e737ef5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:09:57 -0700 Subject: [PATCH 201/260] fix(test): bind Markdown privacy fixtures to active provenance --- ...MarkdownMeasurementErrorPrivacyContract.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index c069b76f..3e172ebb 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { existsSync, mkdtempSync, @@ -11,8 +11,16 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function sha256(value: string) { From aba2d3be70830dfd53c31038573842dcfe36cdd3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:11:55 -0700 Subject: [PATCH 202/260] fix(test): bind HTML serialization fixtures to active provenance --- ...performanceHtmlSerializationMeasurement.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts index e7d016c2..7eaff65e 100644 --- a/src/performanceHtmlSerializationMeasurement.test.ts +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, @@ -14,8 +14,16 @@ const measurementScript = resolve( process.cwd(), 'benchmarks/measure-markdown.mjs', ); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function sha256(path: string): string { From d334b959eeb132b17afc1d9a41c139e0ad66cf10 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 04:45:03 -0700 Subject: [PATCH 203/260] fix(test): bind HTML suite runtime to active provenance --- src/performanceHtmlSerializationSuiteContract.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index e4a6eadb..98dd9389 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -21,6 +21,7 @@ const currentSourceCommitSha = execFileSync( stdio: ['ignore', 'pipe', 'pipe'], }, ).trim(); +const currentRuntimeId = `node-${process.versions.node}`; function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); @@ -78,7 +79,7 @@ describe('single-command HTML serialization benchmark contract', () => { '--revision-artifact-sha256', sha256(revisionModuleSource), '--runtime-id', - 'node-22.0.0', + currentRuntimeId, '--reference-hardware-id', `refhw-sha256-${'b'.repeat(64)}`, '--output', @@ -146,4 +147,4 @@ describe('single-command HTML serialization benchmark contract', () => { rmSync(directory, { recursive: true, force: true }); } }); -}); +}); \ No newline at end of file From fc54c8c30872354fc6ac6bbf0b4803445d17a40c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:09:55 -0700 Subject: [PATCH 204/260] test(perf): reject source movement during Office measurement --- .../test_performance_source_stability.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 office/tests/test_performance_source_stability.py diff --git a/office/tests/test_performance_source_stability.py b/office/tests/test_performance_source_stability.py new file mode 100644 index 00000000..0c51cb6e --- /dev/null +++ b/office/tests/test_performance_source_stability.py @@ -0,0 +1,176 @@ +"""Source-stability contracts for Office benchmark provenance.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + + +def _initialize_clean_repository(repository: Path) -> Path: + repository.mkdir() + _git(repository, "init", "-q") + tracked = repository / "tracked.txt" + tracked.write_text("before\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "initial", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return tracked + + +def test_office_measurement_rejects_clean_source_revision_move_during_sampling( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Never emit benchmark evidence if a clean checkout advances while samples are acquired.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_source_stability") + main = namespace["_main"] + globals_ = main.__globals__ + + repository = tmp_path / "isolated-repository" + tracked = _initialize_clean_repository(repository) + globals_["REPOSITORY_ROOT"] = repository + + request_path = tmp_path / "synthetic-docx-small.json" + request_path.write_bytes(_canonical_docx_small_fixture_bytes()) + + def _move_revision( + _payload_bytes: bytes, + format_name: str, + _profile: str, + ) -> dict[str, object]: + tracked.write_text("after\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "advance", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return {"format": format_name, "durationMs": 1.0, "peakRssBytes": 1024} + + globals_["_run_sample"] = _move_revision + + result = main( + [ + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + "1", + "--reference-hardware", + "pytest-reference", + ] + ) + output = capsys.readouterr() + + assert result == 2 + assert output.out == "" + assert "benchmark source revision changed during measurement" in output.err + assert str(repository) not in output.err + assert str(request_path) not in output.err From 227418d76b51a664a003ea66bbd7320325eab52d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:11:50 -0700 Subject: [PATCH 205/260] fix(perf): bind Office evidence to stable source revision --- office/benchmarks/measure_render.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 4789d264..a2a62e76 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -286,6 +286,9 @@ def _main(argv: list[str] | None = None) -> int: Path(args.input), expected_bytes, expected_sha256 ) samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] + observed_source_sha = _source_sha() + if observed_source_sha != source_sha: + raise BenchmarkContractError("benchmark source revision changed during measurement") duration_values = [float(sample["durationMs"]) for sample in samples] rss_values = [float(sample["peakRssBytes"]) for sample in samples] evidence = { From 483c3fc54b8cecfd2ce18bb64a313d86e2230ebe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:18:50 -0700 Subject: [PATCH 206/260] test(perf): reject source movement during direct measurement --- ...entProducerSourceStabilityContract.test.ts | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/performanceMeasurementProducerSourceStabilityContract.test.ts diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts new file mode 100644 index 00000000..2c3f402b --- /dev/null +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -0,0 +1,167 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const firstSourceSha = 'a'.repeat(40); +const movedSourceSha = 'b'.repeat(40); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'c'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function movingGitEnvironment(root: string): NodeJS.ProcessEnv { + const fakeBin = join(root, 'fake-bin'); + const statePath = join(root, 'git-invocations.txt'); + const fakeGit = join(fakeBin, 'git'); + const script = `#!/usr/bin/env node +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +const state = process.env.INKSPAN_FAKE_GIT_STATE; +const first = process.env.INKSPAN_FAKE_GIT_FIRST_SHA; +const moved = process.env.INKSPAN_FAKE_GIT_MOVED_SHA; +const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; +writeFileSync(state, String(count + 1), 'utf8'); +process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); +`; + writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); + chmodSync(fakeGit, 0o755); + return { + ...process.env, + PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_FAKE_GIT_STATE: statePath, + INKSPAN_FAKE_GIT_FIRST_SHA: firstSourceSha, + INKSPAN_FAKE_GIT_MOVED_SHA: movedSourceSha, + }; +} + +function markdownInvocation(root: string) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Source movement fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +function revisionInvocation(root: string) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'d'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Source movement fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +describe.skipIf(process.platform === 'win32')( + 'direct benchmark producer source stability', + () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects %s evidence when checked-out HEAD moves during sample acquisition', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-move-')); + try { + const { output, result } = invoke(root); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + }, +); From 65e6327cb70b1828cce514fd25f84f1e131538cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:22:13 -0700 Subject: [PATCH 207/260] test(perf): build isolated git shim for source-move RED --- ...erformanceMeasurementProducerSourceStabilityContract.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts index 2c3f402b..12cfdd9e 100644 --- a/src/performanceMeasurementProducerSourceStabilityContract.test.ts +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process'; import { chmodSync, existsSync, + mkdirSync, mkdtempSync, rmSync, writeFileSync, @@ -42,6 +43,7 @@ const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; writeFileSync(state, String(count + 1), 'utf8'); process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); `; + mkdirSync(fakeBin, { recursive: true }); writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); chmodSync(fakeGit, 0o755); return { From 8a7e49c8e669a396b9b7587b2361dc9e33940bc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:04:31 -0700 Subject: [PATCH 208/260] fix(perf): recheck source provenance after markdown measurement --- benchmarks/measure-markdown.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 14904b70..049b1df2 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -454,6 +454,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { From a5123c7b0aa153d7e6eccd82ee07bc625af4ca7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:07:01 -0700 Subject: [PATCH 209/260] fix(perf): recheck source provenance after revision measurement --- benchmarks/measure-revision-evidence.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 782063cb..1e3e438d 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -406,6 +406,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { From 0fc19a7429169b4d09c5d19d9a49f0cdb25ca6df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 17:17:33 -0700 Subject: [PATCH 210/260] test(perf): reject symlinked corpus output directory --- src/performanceCorpusContract.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 02bc5c02..99cc5f1b 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, @@ -108,4 +109,24 @@ describe('deterministic synthetic performance corpus', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of following a corpus output-directory symlink', () => { + const root = mkdtempSync( + join(tmpdir(), 'inkspan-benchmark-corpus-directory-symlink-'), + ); + const outputDirectory = join(root, 'output'); + const buyerOwnedDirectory = join(root, 'buyer-owned'); + const sentinelPath = join(buyerOwnedDirectory, 'sentinel.txt'); + try { + mkdirSync(buyerOwnedDirectory, { recursive: true }); + writeFileSync(sentinelPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(buyerOwnedDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(buyerOwnedDirectory)).toEqual(['sentinel.txt']); + expect(readFileSync(sentinelPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 30edf110bef18e14fe4afc3d662d24e7b1b58b6f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 17:20:46 -0700 Subject: [PATCH 211/260] fix(perf): reject unsafe corpus directory ancestry --- benchmarks/generate-corpus.mjs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index af9d85e9..d54caf0a 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -6,7 +6,7 @@ import { rmSync, writeFileSync, } from 'node:fs'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; const RASTER_FIXTURES = Object.freeze([ Object.freeze({ @@ -50,6 +50,23 @@ const SCRIPT_LABELS = Object.freeze([ let outputWriteCounter = 0; +function assertSafeDirectoryChain(directoryPath) { + let currentPath = directoryPath; + while (true) { + const existing = lstatSync(currentPath, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isDirectory()) { + throw new Error( + 'Benchmark corpus output directory must not traverse symbolic links or non-directories.', + ); + } + const parentPath = dirname(currentPath); + if (parentPath === currentPath) { + break; + } + currentPath = parentPath; + } +} + function writeRegularOutput(path, content) { const existing = lstatSync(path, { throwIfNoEntry: false }); if (existing !== undefined && !existing.isFile()) { @@ -129,7 +146,9 @@ function resolveOutputDirectory(argv) { } const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +assertSafeDirectoryChain(outputDirectory); mkdirSync(outputDirectory, { recursive: true }); +assertSafeDirectoryChain(outputDirectory); const profileManifest = {}; for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { From b5ad1da4de702d716e0e75eae2d26d5697e51fbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 14:08:03 +0900 Subject: [PATCH 212/260] test(perf): keep benchmark contracts portable on Node 24 --- ...performanceMeasurementStatisticsContract.test.ts | 13 ++++++++++++- src/performancePackedArtifactSuiteContract.test.ts | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 7f7627b3..9a4646d4 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -198,7 +198,18 @@ describe('deterministic benchmark sample statistics', () => { truncateSync(input, 16 * 1024 * 1024 + 1); writeFileSync( preload, - `import fs from 'node:fs';\nimport { syncBuiltinESMExports } from 'node:module';\nfs.readFileSync = () => { throw new Error('benchmark whole-file read sentinel'); };\nsyncBuiltinESMExports();\n`, + `import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +const originalReadFileSync = fs.readFileSync.bind(fs); +const blockedInputPath = ${JSON.stringify(input)}; +fs.readFileSync = (path, ...argumentsList) => { + if (path === blockedInputPath) { + throw new Error('benchmark whole-file read sentinel'); + } + return originalReadFileSync(path, ...argumentsList); +}; +syncBuiltinESMExports(); +`, 'utf8', ); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 5e3eb674..f9433bdc 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -190,7 +190,7 @@ describe('packed artifact benchmark suite contract', () => { ) as { benchmarkId?: unknown; samples?: unknown[] }; expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); expect(htmlSamples.samples).toHaveLength(2); - }); + }, 20_000); it('rejects a runtime identifier that does not match the active Node process', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-runtime-')); From d025d45d4a09b9b8f7c87528061f334b7fd7a9f8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 14:10:23 +0900 Subject: [PATCH 213/260] test(perf): allow slow benchmark contract processes --- src/performanceHtmlSerializationSuiteContract.test.ts | 4 ++-- src/performancePackedArtifactPathStability.test.ts | 2 +- src/performanceSingleCommandSuiteContract.test.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index 98dd9389..729acdc8 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -146,5 +146,5 @@ describe('single-command HTML serialization benchmark contract', () => { } finally { rmSync(directory, { recursive: true, force: true }); } - }); -}); \ No newline at end of file + }, 20_000); +}); diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 7ff8c0df..744dda28 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -197,5 +197,5 @@ describe('packed artifact benchmark path stability', () => { readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), ) as { artifactSha256: string }; expect(markdownEvidence.artifactSha256).toBe(original.markdownModuleSha256); - }); + }, 20_000); }); diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index ca04e242..83490f29 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -197,7 +197,7 @@ describe('single-command benchmark suite contract', () => { 'utf8', ), ).toContain('revision-evidence-small'); - }); + }, 20_000); it('rejects a claimed source commit that is not the checked-out HEAD', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-source-')); @@ -318,5 +318,5 @@ describe('single-command benchmark suite contract', () => { ); expect(existsSync(join(actualOutputDirectory, 'markdown'))).toBe(false); expect(existsSync(join(actualOutputDirectory, 'revision'))).toBe(false); - }); + }, 20_000); }); From 97514db77ae3ba262bdd85352cfe61cd28dec04c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 04:25:28 -0700 Subject: [PATCH 214/260] test(perf): define autosave enqueue measurement contract --- ...ormanceAutosaveMeasurementContract.test.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 src/performanceAutosaveMeasurementContract.test.ts diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts new file mode 100644 index 00000000..bdf0002b --- /dev/null +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -0,0 +1,128 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-autosave.mjs', +); +const sourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'a'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('autosave enqueue performance measurement', () => { + it('measures deterministic queue admission without persisting document content', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-measure-')); + const modulePath = join(directory, 'autosave.mjs'); + const outputPath = join(directory, 'samples.json'); + const moduleSource = [ + 'export function createDocumentAutosaveQueue(options) {', + ' return Object.freeze({', + ' async enqueue(evidence) {', + ' const result = await options.save(evidence);', + " if (result?.status !== 'saved') throw new Error('save failed');", + ' return Object.freeze({', + " status: 'saved',", + ' strongEntityTag: evidence.revision.strongEntityTag,', + ' });', + ' },', + ' resume() { return false; },', + ' async flush() { return Object.freeze({ state: \'idle\' }); },', + ' async close() { return Object.freeze({ state: \'closed\' }); },', + ' getSnapshot() { return Object.freeze({ state: \'idle\' }); },', + ' });', + '}', + '', + ].join('\n'); + + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputPath, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const outputText = readFileSync(outputPath, 'utf8'); + const evidence = JSON.parse(outputText) as { + benchmarkId?: unknown; + unit?: unknown; + documentProfile?: unknown; + operation?: unknown; + samples?: unknown[]; + provenance?: Record; + }; + expect(evidence).toMatchObject({ + benchmarkId: 'autosave-enqueue-small', + unit: 'ms', + documentProfile: 'small', + operation: 'autosave-enqueue', + provenance: { + sourceCommitSha, + artifactSha256: sha256(moduleSource), + runtimeId, + referenceHardwareId, + }, + }); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples?.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(outputText).not.toContain('Synthetic autosave benchmark document'); + expect(outputText).not.toContain(modulePath); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 20_000); +}); From 690c03968c25fc482d2162b853fb0fdd9c06f6e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 13:11:34 -0700 Subject: [PATCH 215/260] fix(perf): implement autosave enqueue measurement --- benchmarks/measure-autosave.mjs | 448 ++++++++++++++++++++++++++++++++ 1 file changed, 448 insertions(+) create mode 100644 benchmarks/measure-autosave.mjs diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs new file mode 100644 index 00000000..9c4c8fdc --- /dev/null +++ b/benchmarks/measure-autosave.mjs @@ -0,0 +1,448 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Autosave benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = 'Autosave benchmark output must not already exist.'; +const SYNTHETIC_DOCUMENT_LABEL = 'Synthetic autosave benchmark document'; + +function resolveArguments(argv) { + const expectedFlags = [ + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-autosave.mjs --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Autosave benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error( + 'Autosave benchmark sample count must be an integer from 1 to 1000.', + ); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Autosave benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Autosave benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Autosave benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Autosave benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidFileMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured autosave module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured autosave module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured autosave module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured autosave module must be a local regular file.', + 'Measured autosave module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Autosave benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Autosave benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Autosave benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) throw new Error(OUTPUT_DIRECTORY_ERROR); + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Autosave benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured autosave module could not be loaded.'); + } +} + +function createSyntheticRevisionEvidence() { + const textNode = Object.freeze({ type: 'text', text: SYNTHETIC_DOCUMENT_LABEL }); + const paragraph = Object.freeze({ + type: 'paragraph', + content: Object.freeze([textNode]), + }); + const documentJson = Object.freeze({ + type: 'doc', + content: Object.freeze([paragraph]), + }); + const digestHex = createHash('sha256') + .update(JSON.stringify(documentJson)) + .digest('hex'); + return Object.freeze({ + envelope: Object.freeze({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson, + }), + revision: Object.freeze({ + algorithm: 'SHA-256', + digestHex, + strongEntityTag: `"sha256-${digestHex}"`, + }), + }); +} + +async function measureOneEnqueue(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return Object.freeze({ status: 'saved' }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + + const start = performance.now(); + const outcome = await queue.enqueue(evidence); + const elapsed = performance.now() - start; + if ( + typeof outcome !== 'object' || + outcome === null || + outcome.status !== 'saved' || + saveCalls !== 1 + ) { + throw new Error('Measured autosave enqueue result is invalid.'); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Autosave measurement produced invalid runtime evidence.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Autosave benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Autosave benchmark output could not be written.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Autosave benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + if (typeof measuredModule.createDocumentAutosaveQueue !== 'function') { + throw new Error( + 'Measured autosave module must export createDocumentAutosaveQueue().', + ); + } + const evidence = createSyntheticRevisionEvidence(); + + await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence); + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + samples.push( + await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence), + ); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Autosave benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Autosave benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) throw new Error(OUTPUT_EXISTS_ERROR); + + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `autosave-enqueue-${args.profile}`, + unit: 'ms', + documentProfile: args.profile, + operation: 'autosave-enqueue', + samples, + provenance: { + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + }, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Autosave benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From 8fc0ba45339e3fc7209701efa31eaa12c1872ebd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 15:42:54 -0700 Subject: [PATCH 216/260] test(perf): require bounded PR benchmark evidence workflow --- src/performanceWorkflowContract.test.ts | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/performanceWorkflowContract.test.ts diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts new file mode 100644 index 00000000..30e52b09 --- /dev/null +++ b/src/performanceWorkflowContract.test.ts @@ -0,0 +1,50 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const workflowPath = resolve( + repositoryRoot, + '.github/workflows/performance-evidence.yml', +); + +describe('performance evidence workflow contract', () => { + it('runs a bounded exact-head packed-artifact benchmark on performance-relevant PRs', () => { + expect(existsSync(workflowPath)).toBe(true); + if (!existsSync(workflowPath)) return; + + const workflow = readFileSync(workflowPath, 'utf8'); + + expect(workflow).toContain('name: Performance Evidence'); + expect(workflow).toContain('pull_request:'); + expect(workflow).toContain('permissions:\n contents: read'); + expect(workflow).toContain('runs-on: ubuntu-24.04'); + expect(workflow).toContain('timeout-minutes: 20'); + expect(workflow).toContain( + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', + ); + expect(workflow).toContain( + 'ref: ${{ github.event.pull_request.head.sha }}', + ); + expect(workflow).toContain('persist-credentials: false'); + expect(workflow).toContain('Verify exact checkout'); + expect(workflow).toContain('pnpm install --frozen-lockfile'); + expect(workflow).toContain('pnpm build'); + expect(workflow).toContain('pnpm pack --pack-destination'); + expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); + expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); + expect(workflow).toContain('--profile small'); + expect(workflow).toContain('--samples 3'); + expect(workflow).toContain( + '--reference-hardware-id github-actions-ubuntu-24.04-x64', + ); + expect(workflow).toContain( + 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a', + ); + expect(workflow).toContain('retention-days: 5'); + expect(workflow).not.toContain('secrets.'); + expect(workflow).not.toContain('contents: write'); + expect(workflow).not.toContain('pull-requests: write'); + }); +}); From 55a56bc0b30b0528fe99018d677f7535e763a976 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 15:46:54 -0700 Subject: [PATCH 217/260] ci(perf): add bounded exact-head smoke evidence --- .github/workflows/performance-evidence.yml | 102 +++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/performance-evidence.yml diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml new file mode 100644 index 00000000..2ccbb803 --- /dev/null +++ b/.github/workflows/performance-evidence.yml @@ -0,0 +1,102 @@ +name: Performance Evidence + +on: + pull_request: + paths: + - '.github/workflows/performance-evidence.yml' + - 'benchmarks/**' + - 'src/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig.json' + - 'vite.config.ts' + +permissions: + contents: read + +concurrency: + group: performance-evidence-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + packed-artifact-smoke: + name: packed-artifact-smoke + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout exact pull-request head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Verify exact checkout + shell: bash + env: + INKSPAN_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + actual_head="$(git rev-parse HEAD)" + test "$actual_head" = "$INKSPAN_EXPECTED_HEAD_SHA" + + - name: Set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: pnpm + + - name: Install immutable dependencies + run: pnpm install --frozen-lockfile + + - name: Build packed library + run: pnpm build + + - name: Produce bounded packed-artifact smoke evidence + shell: bash + run: | + set -euo pipefail + corpus_dir="${RUNNER_TEMP}/inkspan-benchmark-corpus" + revision_input="${RUNNER_TEMP}/inkspan-revision-input.json" + package_dir="${RUNNER_TEMP}/inkspan-package" + evidence_dir="${RUNNER_TEMP}/inkspan-performance-evidence" + + rm -rf "$corpus_dir" "$package_dir" "$evidence_dir" + rm -f "$revision_input" + + node benchmarks/generate-corpus.mjs --output "$corpus_dir" + printf '%s\n' '{"contractVersion":1,"mode":"markdown","document":"# Inkspan deterministic performance smoke"}' > "$revision_input" + + mkdir -p "$package_dir" + pnpm pack --pack-destination "$package_dir" + mapfile -t packages < <(find "$package_dir" -maxdepth 1 -type f -name '*.tgz' -print) + test "${#packages[@]}" -eq 1 + package="${packages[0]}" + package_sha256="$(sha256sum "$package" | cut -d' ' -f1)" + source_sha="$(git rev-parse HEAD)" + runtime_id="node-$(node -p 'process.versions.node')" + + node benchmarks/run-current-suite.mjs \ + --input "$corpus_dir/small.md" \ + --revision-input "$revision_input" \ + --package-tarball "$package" \ + --package-sha256 "$package_sha256" \ + --profile small \ + --samples 3 \ + --source-commit-sha "$source_sha" \ + --runtime-id "$runtime_id" \ + --reference-hardware-id github-actions-ubuntu-24.04-x64 \ + --output "$evidence_dir" + + - name: Upload bounded performance evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0.0 + with: + name: performance-smoke-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/inkspan-performance-evidence + if-no-files-found: error + retention-days: 5 From 0ad936601d5130fdead20d9b1b7383d092fefa22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 15:50:05 -0700 Subject: [PATCH 218/260] test(perf): bind smoke revision input to envelope schema --- src/performanceWorkflowContract.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index 30e52b09..24ba1ce2 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -33,6 +33,11 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain('pnpm build'); expect(workflow).toContain('pnpm pack --pack-destination'); expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); + expect(workflow).toContain( + '"schemaId":"https://inkspan.io/schemas/document-envelope/v1"', + ); + expect(workflow).toContain('"schemaVersion":1'); + expect(workflow).toContain('"documentJson":{"type":"doc"'); expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); expect(workflow).toContain('--profile small'); expect(workflow).toContain('--samples 3'); From a5b741b99d471aa9f2419d539bb4f182c89d3ea4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 15:50:40 -0700 Subject: [PATCH 219/260] fix(perf): benchmark a valid document envelope --- .github/workflows/performance-evidence.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index 2ccbb803..4204a886 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -70,7 +70,7 @@ jobs: rm -f "$revision_input" node benchmarks/generate-corpus.mjs --output "$corpus_dir" - printf '%s\n' '{"contractVersion":1,"mode":"markdown","document":"# Inkspan deterministic performance smoke"}' > "$revision_input" + printf '%s\n' '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Inkspan deterministic performance smoke"}]}]}}' > "$revision_input" mkdir -p "$package_dir" pnpm pack --pack-destination "$package_dir" From 2d3acdd01b72a47fc1e2e4bc0e99b61c0afcb2c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:03:39 -0700 Subject: [PATCH 220/260] test(perf): require packed autosave suite evidence --- ...ormancePackedArtifactSuiteContract.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index f9433bdc..97dc8ca6 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -71,6 +71,19 @@ function createPackedBenchmarkFixture(directory: string): { `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`, 'utf8', ); + writeFileSync( + join(distDirectory, 'cwl-autosave.js'), + [ + 'export function createDocumentAutosaveQueue({ save }) {', + ' return {', + ' async enqueue(evidence) { return await save(evidence); },', + ' async close() {},', + ' };', + '}', + '', + ].join('\n'), + 'utf8', + ); const packResult = JSON.parse( execFileSync( @@ -174,6 +187,9 @@ describe('packed artifact benchmark suite contract', () => { packageName: '@contextualwisdomlab/cwl-editor', packageVersion: '0.0.0-benchmark-fixture', packageSha256: packed.packageSha256, + autosaveSamples: 'autosave/samples.json', + autosaveSummaryJson: 'autosave/summary/summary.json', + autosaveSummaryText: 'autosave/summary/summary.txt', htmlSerializationSamples: 'html-serialization/samples.json', htmlSerializationSummaryJson: 'html-serialization/summary/summary.json', @@ -182,6 +198,15 @@ describe('packed artifact benchmark suite contract', () => { status: 'completed', }); + const autosaveSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(autosaveSamples.benchmarkId).toBe('autosave-enqueue-small'); + expect(autosaveSamples.samples).toHaveLength(2); + const htmlSamples = JSON.parse( readFileSync( join(directory, 'evidence', 'html-serialization', 'samples.json'), From 4cb6c497ae799f9c5441426cad736cad1730de55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:05:43 -0700 Subject: [PATCH 221/260] feat(perf): include autosave in packed benchmark suite --- benchmarks/run-current-suite-core.mjs | 76 +++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index a18479e8..fee54756 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -56,6 +56,7 @@ const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; +const AUTOSAVE_MODULE_ENTRY = 'package/dist/cwl-autosave.js'; const OUTPUT_DIRECTORY_ERROR = 'Benchmark suite output directory must be a non-symlink directory.'; const OUTPUT_DIRECTORY_EXISTS_ERROR = @@ -114,6 +115,25 @@ function measurementArguments({ ]); } +function autosaveMeasurementArguments({ modulePath, artifactSha256, shared }) { + return Object.freeze([ + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + function currentCheckoutSha() { const result = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repositoryRoot, @@ -401,6 +421,7 @@ function preparePackedBenchmarkModules(args) { PACKAGE_MANIFEST_ENTRY, MARKDOWN_MODULE_ENTRY, REVISION_MODULE_ENTRY, + AUTOSAVE_MODULE_ENTRY, ]) { assertUniquePackedEntry(entries, entry); } @@ -421,6 +442,11 @@ function preparePackedBenchmarkModules(args) { REVISION_MODULE_ENTRY, MAX_MODULE_BYTES, ); + const autosaveModuleBytes = readPackedEntry( + args.packageTarballPath, + AUTOSAVE_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); verifyPackageDigest(args.packageTarballPath, args.packageSha256); const temporaryDirectory = mkdtempSync( @@ -431,9 +457,11 @@ function preparePackedBenchmarkModules(args) { temporaryDirectory, 'cwl-revision-evidence.mjs', ); + const autosaveModulePath = join(temporaryDirectory, 'cwl-autosave.mjs'); try { writeFileSync(markdownModulePath, markdownModuleBytes); writeFileSync(revisionModulePath, revisionModuleBytes); + writeFileSync(autosaveModulePath, autosaveModuleBytes); } catch { rmSync(temporaryDirectory, { recursive: true, force: true }); throw new Error('Benchmark suite package modules could not be prepared.'); @@ -445,6 +473,8 @@ function preparePackedBenchmarkModules(args) { markdownArtifactSha256: moduleSha256(markdownModuleBytes), revisionModulePath, revisionArtifactSha256: moduleSha256(revisionModuleBytes), + autosaveModulePath, + autosaveArtifactSha256: moduleSha256(autosaveModuleBytes), packageEvidence: Object.freeze({ packageName: manifest.name, packageVersion: manifest.version, @@ -495,7 +525,7 @@ function runMeasurementAndSummary({ ); } -function runSuite(args, markdownArguments, revisionArguments) { +function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) { const markdownSamplesPath = resolve( args.outputDirectory, 'markdown', @@ -533,9 +563,30 @@ function runSuite(args, markdownArguments, revisionArguments) { measurementFailure: 'Benchmark suite revision measurement failed.', summaryFailure: 'Benchmark suite revision summary failed.', }); + + if (autosaveArguments !== null) { + const autosaveSamplesPath = resolve( + args.outputDirectory, + 'autosave', + 'samples.json', + ); + const autosaveSummaryDirectory = resolve( + args.outputDirectory, + 'autosave', + 'summary', + ); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: autosaveArguments, + samplesPath: autosaveSamplesPath, + summaryDirectory: autosaveSummaryDirectory, + measurementFailure: 'Benchmark suite autosave measurement failed.', + summaryFailure: 'Benchmark suite autosave summary failed.', + }); + } } -function suiteManifest(args, packageEvidence) { +function suiteManifest(args, packageEvidence, includeAutosave) { return Object.freeze({ contractVersion: 1, documentProfile: args.documentProfile, @@ -550,6 +601,13 @@ function suiteManifest(args, packageEvidence) { revisionSamples: 'revision/samples.json', revisionSummaryJson: 'revision/summary/summary.json', revisionSummaryText: 'revision/summary/summary.txt', + ...(includeAutosave + ? { + autosaveSamples: 'autosave/samples.json', + autosaveSummaryJson: 'autosave/summary/summary.json', + autosaveSummaryText: 'autosave/summary/summary.txt', + } + : {}), status: 'completed', }); } @@ -580,13 +638,21 @@ function main(argv) { shared, }) : resolved.revisionArguments; + const autosaveArguments = + resolved.mode === 'packed' + ? autosaveMeasurementArguments({ + modulePath: preparedPackage.autosaveModulePath, + artifactSha256: preparedPackage.autosaveArtifactSha256, + shared, + }) + : null; const packageEvidence = resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; let createdOutputDirectory = false; try { createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); - runSuite(shared, markdownArguments, revisionArguments); + runSuite(shared, markdownArguments, revisionArguments, autosaveArguments); if (resolved.mode === 'packed') { verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); } @@ -605,7 +671,9 @@ function main(argv) { } process.stdout.write( - `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, + `${JSON.stringify( + suiteManifest(shared, packageEvidence, resolved.mode === 'packed'), + )}\n`, ); } From 4a68ae301d5f695ff0e407832b8549984adf16ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:08:56 -0700 Subject: [PATCH 222/260] test(perf): require canonical autosave sample schema --- ...ormanceAutosaveMeasurementContract.test.ts | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index bdf0002b..d63ae39a 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -32,7 +32,7 @@ function sha256(source: string): string { } describe('autosave enqueue performance measurement', () => { - it('measures deterministic queue admission without persisting document content', () => { + it('emits the canonical summarizable sample contract without persisting document content', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-measure-')); const modulePath = join(directory, 'autosave.mjs'); const outputPath = join(directory, 'samples.json'); @@ -93,25 +93,39 @@ describe('autosave enqueue performance measurement', () => { const outputText = readFileSync(outputPath, 'utf8'); const evidence = JSON.parse(outputText) as { + contractVersion?: unknown; benchmarkId?: unknown; unit?: unknown; + sourceCommitSha?: unknown; + artifactSha256?: unknown; documentProfile?: unknown; - operation?: unknown; + runtimeId?: unknown; + referenceHardwareId?: unknown; samples?: unknown[]; - provenance?: Record; }; expect(evidence).toMatchObject({ + contractVersion: 1, benchmarkId: 'autosave-enqueue-small', unit: 'ms', + sourceCommitSha, + artifactSha256: sha256(moduleSource), documentProfile: 'small', - operation: 'autosave-enqueue', - provenance: { - sourceCommitSha, - artifactSha256: sha256(moduleSource), - runtimeId, - referenceHardwareId, - }, + runtimeId, + referenceHardwareId, }); + expect(Object.keys(evidence).sort()).toEqual( + [ + 'artifactSha256', + 'benchmarkId', + 'contractVersion', + 'documentProfile', + 'referenceHardwareId', + 'runtimeId', + 'samples', + 'sourceCommitSha', + 'unit', + ].sort(), + ); expect(evidence.samples).toHaveLength(2); expect( evidence.samples?.every( From 36ba0fa7d3bac9990ff4edd2ec6c8cd51767f6c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:10:05 -0700 Subject: [PATCH 223/260] fix(perf): standardize autosave benchmark evidence --- benchmarks/measure-autosave.mjs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 9c4c8fdc..055e7e81 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -420,15 +420,12 @@ async function main() { contractVersion: 1, benchmarkId: `autosave-enqueue-${args.profile}`, unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, documentProfile: args.profile, - operation: 'autosave-enqueue', + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, samples, - provenance: { - sourceCommitSha: args.sourceCommitSha, - artifactSha256: args.artifactSha256, - runtimeId: args.runtimeId, - referenceHardwareId: args.referenceHardwareId, - }, }, null, 2, From b3a1aff515ce45dd0c0b2143c943869766b4a790 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:13:34 -0700 Subject: [PATCH 224/260] test(perf): require hosted HTML serialization evidence --- src/performanceWorkflowContract.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index 24ba1ce2..7eabc232 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -33,6 +33,13 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain('pnpm build'); expect(workflow).toContain('pnpm pack --pack-destination'); expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); + expect(workflow).toContain( + 'html_input="${RUNNER_TEMP}/inkspan-html-input.html"', + ); + expect(workflow).toContain( + "printf '%s\\n' '

Inkspan deterministic performance smoke

' > \"$html_input\"", + ); + expect(workflow).toContain('--html-input "$html_input"'); expect(workflow).toContain( '"schemaId":"https://inkspan.io/schemas/document-envelope/v1"', ); From f88ba74c91427d26b3fc59a7ac31c914f4981075 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 17:17:24 -0700 Subject: [PATCH 225/260] ci(perf): smoke HTML serialization evidence --- .github/workflows/performance-evidence.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index 4204a886..97a72c8e 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -62,14 +62,16 @@ jobs: run: | set -euo pipefail corpus_dir="${RUNNER_TEMP}/inkspan-benchmark-corpus" + html_input="${RUNNER_TEMP}/inkspan-html-input.html" revision_input="${RUNNER_TEMP}/inkspan-revision-input.json" package_dir="${RUNNER_TEMP}/inkspan-package" evidence_dir="${RUNNER_TEMP}/inkspan-performance-evidence" rm -rf "$corpus_dir" "$package_dir" "$evidence_dir" - rm -f "$revision_input" + rm -f "$html_input" "$revision_input" node benchmarks/generate-corpus.mjs --output "$corpus_dir" + printf '%s\n' '

Inkspan deterministic performance smoke

' > "$html_input" printf '%s\n' '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Inkspan deterministic performance smoke"}]}]}}' > "$revision_input" mkdir -p "$package_dir" @@ -83,6 +85,7 @@ jobs: node benchmarks/run-current-suite.mjs \ --input "$corpus_dir/small.md" \ + --html-input "$html_input" \ --revision-input "$revision_input" \ --package-tarball "$package" \ --package-sha256 "$package_sha256" \ From e6659b2892774fb4a36fa1803b79e3e4eb7cedf3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 19:04:33 -0700 Subject: [PATCH 226/260] fix(perf): keep path-stability fixture current with packed suite --- src/performancePackedArtifactPathStability.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 744dda28..6c1aa315 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -75,6 +75,19 @@ function createPackedBenchmarkFixture( `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'e'.repeat(64)}' } }; }\n`, 'utf8', ); + writeFileSync( + join(distDirectory, 'cwl-autosave.js'), + [ + 'export function createDocumentAutosaveQueue({ save }) {', + ' return {', + ' async enqueue(evidence) { return await save(evidence); },', + ' async close() {},', + ' };', + '}', + '', + ].join('\n'), + 'utf8', + ); const packResult = JSON.parse( execFileSync( From 3d5e739c81e0e8c24d55459f5c87c3fd3faea96e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:51:46 +0900 Subject: [PATCH 227/260] test(perf): canonicalize the macOS temp root --- test/setup.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/setup.ts b/test/setup.ts index 5bf253a3..4ecf96d7 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1,4 +1,11 @@ import '@testing-library/jest-dom/vitest'; +import { realpathSync } from 'node:fs'; +import { tmpdir } from 'node:os'; + +// macOS exposes its system temp directory through /var -> /private/var. Give +// child-process path guards the canonical platform path so they can still +// reject symlinks created inside the test boundary. +process.env.TMPDIR = realpathSync(tmpdir()); // jsdom does not implement canvas; the downscale path falls back gracefully, // but stubbing getContext keeps any accidental calls from throwing. From a9804ae74fc2d05ad39b3c9499759cd1135d9a51 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:24:05 +0900 Subject: [PATCH 228/260] fix(ci): preserve Python boundary coverage Signed-off-by: Seongho Bae (cherry picked from commit 870c2c3efffced3ce8d280b4487e4eda055c1ff1) --- .github/workflows/ci.yml | 2 +- office/tests/test_python_support_contract.py | 14 ++++++++++---- src/workflowExactHead.test.ts | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f7614e5..03a50567 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.14"]') || fromJSON('["3.11", "3.12", "3.13", "3.14"]') }} + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.11", "3.14"]') || fromJSON('["3.11", "3.12", "3.13", "3.14"]') }} defaults: run: working-directory: office diff --git a/office/tests/test_python_support_contract.py b/office/tests/test_python_support_contract.py index 7104fd66..9c9d2288 100644 --- a/office/tests/test_python_support_contract.py +++ b/office/tests/test_python_support_contract.py @@ -1,12 +1,14 @@ """Cross-file contract for the Python versions advertised by Inkspan Office.""" from pathlib import Path +import json import re import tomllib REPOSITORY_ROOT = Path(__file__).resolve().parents[2] SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14") +PULL_REQUEST_PYTHON_VERSIONS = ("3.11", "3.14") PYTHON_312_LXML_LINUX_SHA256 = ( "bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814" ) @@ -33,7 +35,7 @@ def _workflow_job_block(workflow: str, job_name: str) -> str: def test_python_support_range_matches_classifiers_and_ci_matrix() -> None: - """Require package metadata and the Office CI job to cover the same minors.""" + """Require PR boundary coverage and full protected-main compatibility coverage.""" pyproject = tomllib.loads(_repository_text("office/pyproject.toml")) project = pyproject["project"] @@ -50,10 +52,14 @@ def test_python_support_range_matches_classifiers_and_ci_matrix() -> None: office_job = _workflow_job_block(workflow, "office") assert "runs-on: ubuntu-24.04" in office_job assert "runs-on: ubuntu-latest" not in office_job - matrix_match = re.search(r'python-version:\s*\[([^\]]+)\]', office_job) + matrix_match = re.search( + r"python-version:\s*\$\{\{\s*github\.event_name == 'pull_request'\s*" + r"&&\s*fromJSON\('([^']+)'\)\s*\|\|\s*fromJSON\('([^']+)'\)\s*\}\}", + office_job, + ) assert matrix_match is not None - matrix_versions = tuple(re.findall(r'"(3\.\d+)"', matrix_match.group(1))) - assert matrix_versions == SUPPORTED_PYTHON_VERSIONS + assert tuple(json.loads(matrix_match.group(1))) == PULL_REQUEST_PYTHON_VERSIONS + assert tuple(json.loads(matrix_match.group(2))) == SUPPORTED_PYTHON_VERSIONS def test_python_support_documentation_matches_the_fixed_ci_environment() -> None: diff --git a/src/workflowExactHead.test.ts b/src/workflowExactHead.test.ts index 828c2828..404fc623 100644 --- a/src/workflowExactHead.test.ts +++ b/src/workflowExactHead.test.ts @@ -74,7 +74,7 @@ describe('exact-head CI workflow contract', () => { ); expect(workflow).toContain('cancel-in-progress: true'); expect(officeJob).toContain( - "python-version: ${{ github.event_name == 'pull_request' && fromJSON('[\"3.14\"]') || fromJSON('[\"3.11\", \"3.12\", \"3.13\", \"3.14\"]') }}", + "python-version: ${{ github.event_name == 'pull_request' && fromJSON('[\"3.11\", \"3.14\"]') || fromJSON('[\"3.11\", \"3.12\", \"3.13\", \"3.14\"]') }}", ); expect(releaseWorkflow).toContain( 'group: ${{ github.workflow }}-${{ github.repository }}-${{ github.ref_name }}', From d22bcfc6d48708eef6ab1b3c169ea9f425383987 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:31:58 +0900 Subject: [PATCH 229/260] fix(ci): restore full Python PR matrix Signed-off-by: Seongho Bae (cherry picked from commit 93fd077adc8eb5c6980dc47af7cedaef2f211537) --- .github/workflows/ci.yml | 2 +- office/tests/test_python_support_contract.py | 14 ++++---------- src/workflowExactHead.test.ts | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03a50567..eb28caf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.11", "3.14"]') || fromJSON('["3.11", "3.12", "3.13", "3.14"]') }} + python-version: ["3.11", "3.12", "3.13", "3.14"] defaults: run: working-directory: office diff --git a/office/tests/test_python_support_contract.py b/office/tests/test_python_support_contract.py index 9c9d2288..7104fd66 100644 --- a/office/tests/test_python_support_contract.py +++ b/office/tests/test_python_support_contract.py @@ -1,14 +1,12 @@ """Cross-file contract for the Python versions advertised by Inkspan Office.""" from pathlib import Path -import json import re import tomllib REPOSITORY_ROOT = Path(__file__).resolve().parents[2] SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14") -PULL_REQUEST_PYTHON_VERSIONS = ("3.11", "3.14") PYTHON_312_LXML_LINUX_SHA256 = ( "bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814" ) @@ -35,7 +33,7 @@ def _workflow_job_block(workflow: str, job_name: str) -> str: def test_python_support_range_matches_classifiers_and_ci_matrix() -> None: - """Require PR boundary coverage and full protected-main compatibility coverage.""" + """Require package metadata and the Office CI job to cover the same minors.""" pyproject = tomllib.loads(_repository_text("office/pyproject.toml")) project = pyproject["project"] @@ -52,14 +50,10 @@ def test_python_support_range_matches_classifiers_and_ci_matrix() -> None: office_job = _workflow_job_block(workflow, "office") assert "runs-on: ubuntu-24.04" in office_job assert "runs-on: ubuntu-latest" not in office_job - matrix_match = re.search( - r"python-version:\s*\$\{\{\s*github\.event_name == 'pull_request'\s*" - r"&&\s*fromJSON\('([^']+)'\)\s*\|\|\s*fromJSON\('([^']+)'\)\s*\}\}", - office_job, - ) + matrix_match = re.search(r'python-version:\s*\[([^\]]+)\]', office_job) assert matrix_match is not None - assert tuple(json.loads(matrix_match.group(1))) == PULL_REQUEST_PYTHON_VERSIONS - assert tuple(json.loads(matrix_match.group(2))) == SUPPORTED_PYTHON_VERSIONS + matrix_versions = tuple(re.findall(r'"(3\.\d+)"', matrix_match.group(1))) + assert matrix_versions == SUPPORTED_PYTHON_VERSIONS def test_python_support_documentation_matches_the_fixed_ci_environment() -> None: diff --git a/src/workflowExactHead.test.ts b/src/workflowExactHead.test.ts index 404fc623..615f7564 100644 --- a/src/workflowExactHead.test.ts +++ b/src/workflowExactHead.test.ts @@ -74,7 +74,7 @@ describe('exact-head CI workflow contract', () => { ); expect(workflow).toContain('cancel-in-progress: true'); expect(officeJob).toContain( - "python-version: ${{ github.event_name == 'pull_request' && fromJSON('[\"3.11\", \"3.14\"]') || fromJSON('[\"3.11\", \"3.12\", \"3.13\", \"3.14\"]') }}", + 'python-version: ["3.11", "3.12", "3.13", "3.14"]', ); expect(releaseWorkflow).toContain( 'group: ${{ github.workflow }}-${{ github.repository }}-${{ github.ref_name }}', From 8f8cd7df463144cfba49fd8069ef9e5240e54a15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:40:37 +0900 Subject: [PATCH 230/260] fix(perf): bind revision evidence to corpus profiles Signed-off-by: Seongho Bae --- .github/workflows/performance-evidence.yml | 6 ++---- benchmarks/corpus.lock.json | 16 ++++++++++++---- benchmarks/generate-corpus.mjs | 18 ++++++++++++++++++ src/performanceCorpusContract.test.ts | 16 ++++++++++++++++ src/performanceWorkflowContract.test.ts | 4 +--- 5 files changed, 49 insertions(+), 11 deletions(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index 97a72c8e..e173f421 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -63,16 +63,14 @@ jobs: set -euo pipefail corpus_dir="${RUNNER_TEMP}/inkspan-benchmark-corpus" html_input="${RUNNER_TEMP}/inkspan-html-input.html" - revision_input="${RUNNER_TEMP}/inkspan-revision-input.json" package_dir="${RUNNER_TEMP}/inkspan-package" evidence_dir="${RUNNER_TEMP}/inkspan-performance-evidence" rm -rf "$corpus_dir" "$package_dir" "$evidence_dir" - rm -f "$html_input" "$revision_input" + rm -f "$html_input" node benchmarks/generate-corpus.mjs --output "$corpus_dir" printf '%s\n' '

Inkspan deterministic performance smoke

' > "$html_input" - printf '%s\n' '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Inkspan deterministic performance smoke"}]}]}}' > "$revision_input" mkdir -p "$package_dir" pnpm pack --pack-destination "$package_dir" @@ -86,7 +84,7 @@ jobs: node benchmarks/run-current-suite.mjs \ --input "$corpus_dir/small.md" \ --html-input "$html_input" \ - --revision-input "$revision_input" \ + --revision-input "$corpus_dir/small.envelope.json" \ --package-tarball "$package" \ --package-sha256 "$package_sha256" \ --profile small \ diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index fa2fe87c..c0c834b8 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -13,22 +13,30 @@ "small": { "sections": 1, "bytes": 2152, - "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea" + "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea", + "envelopeBytes": 2374, + "envelopeSha256": "0bb4a9ce44d3f93713fec4ef636177bf9693670ba59434fb818b461e8546035a" }, "medium": { "sections": 8, "bytes": 15712, - "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c" + "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c", + "envelopeBytes": 16186, + "envelopeSha256": "240c22c05b8889e3a8e9f211c18227053955d2146c43a25cb010cfee975d6486" }, "large": { "sections": 32, "bytes": 62199, - "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392" + "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392", + "envelopeBytes": 63537, + "envelopeSha256": "e04c53670213b4537243ba8b7c72036b5f0837f40cd150f6947ec38bf9e1b0d4" }, "stress": { "sections": 128, "bytes": 248152, - "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928" + "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928", + "envelopeBytes": 252946, + "envelopeSha256": "858e857576c95e1259b3a817412f29b916fad8df0aa6b104cb39ecc473a5719c" } } } diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index d54caf0a..53bba374 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -134,6 +134,17 @@ function buildProfile(profile, sectionCount) { ].join('\n'); } +function buildEnvelope(body) { + return `${JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [{ type: 'paragraph', content: [{ type: 'text', text: body }] }], + }, + })}\n`; +} + function sha256(bytes) { return createHash('sha256').update(bytes).digest('hex'); } @@ -154,11 +165,18 @@ const profileManifest = {}; for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { const body = buildProfile(profile, sections); const bytes = Buffer.from(body, 'utf8'); + const envelopeBytes = Buffer.from(buildEnvelope(body), 'utf8'); writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); + writeRegularOutput( + resolve(outputDirectory, `${profile}.envelope.json`), + envelopeBytes, + ); profileManifest[profile] = Object.freeze({ sections, bytes: bytes.byteLength, sha256: sha256(bytes), + envelopeBytes: envelopeBytes.byteLength, + envelopeSha256: sha256(envelopeBytes), }); } diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 99cc5f1b..0a5df7e7 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -16,6 +16,8 @@ interface BenchmarkProfileLock { readonly sections: number; readonly bytes: number; readonly sha256: string; + readonly envelopeBytes: number; + readonly envelopeSha256: string; } interface BenchmarkCorpusLock { @@ -77,6 +79,20 @@ describe('deterministic synthetic performance corpus', () => { const secondBytes = readFileSync(join(second, `${profile}.md`)); expect(firstBytes.equals(secondBytes)).toBe(true); expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + const firstEnvelope = readFileSync( + join(first, `${profile}.envelope.json`), + ); + const secondEnvelope = readFileSync( + join(second, `${profile}.envelope.json`), + ); + expect(firstEnvelope.equals(secondEnvelope)).toBe(true); + expect(firstEnvelope.byteLength).toBe( + expected.profiles[profile].envelopeBytes, + ); + expect( + JSON.parse(firstEnvelope.toString('utf8')).documentJson.content[0] + .content[0].text, + ).toBe(firstBytes.toString('utf8')); } const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index 7eabc232..7a37544e 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -41,10 +41,8 @@ describe('performance evidence workflow contract', () => { ); expect(workflow).toContain('--html-input "$html_input"'); expect(workflow).toContain( - '"schemaId":"https://inkspan.io/schemas/document-envelope/v1"', + '--revision-input "$corpus_dir/small.envelope.json"', ); - expect(workflow).toContain('"schemaVersion":1'); - expect(workflow).toContain('"documentJson":{"type":"doc"'); expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); expect(workflow).toContain('--profile small'); expect(workflow).toContain('--samples 3'); From b2cf91faff389d135408154c28b9218588703a89 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:43:59 +0900 Subject: [PATCH 231/260] fix(perf): measure autosave with profile input Signed-off-by: Seongho Bae --- benchmarks/measure-autosave.mjs | 46 +++++++++++++++++-- benchmarks/run-current-suite-core.mjs | 2 + ...ormanceAutosaveMeasurementContract.test.ts | 9 ++++ ...ormancePackedArtifactPathStability.test.ts | 2 +- ...ormancePackedArtifactSuiteContract.test.ts | 2 +- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 055e7e81..0cf723ab 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -19,6 +19,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); @@ -35,7 +36,7 @@ const OUTPUT_EXISTS_ERROR = 'Autosave benchmark output must not already exist.'; const SYNTHETIC_DOCUMENT_LABEL = 'Synthetic autosave benchmark document'; function resolveArguments(argv) { - const expectedFlags = [ + const legacyFlags = [ '--module', '--profile', '--samples', @@ -45,13 +46,16 @@ function resolveArguments(argv) { '--reference-hardware-id', '--output', ]; + const inputFlags = ['--input', ...legacyFlags]; + const expectedFlags = + argv.length === inputFlags.length * 2 ? inputFlags : legacyFlags; if ( argv.length !== expectedFlags.length * 2 || expectedFlags.some((flag, index) => argv[index * 2] !== flag) || expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) ) { throw new Error( - 'Usage: node benchmarks/measure-autosave.mjs --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + 'Usage: node benchmarks/measure-autosave.mjs [--input ] --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } @@ -94,6 +98,8 @@ function resolveArguments(argv) { } return Object.freeze({ + inputPath: + values['--input'] === undefined ? null : resolve(values['--input']), modulePath: values['--module'], profile, sampleCount, @@ -294,7 +300,39 @@ async function loadMeasuredModule(modulePath) { } } -function createSyntheticRevisionEvidence() { +function createSyntheticRevisionEvidence(inputPath) { + if (inputPath !== null) { + const source = readBoundedRegularFile( + inputPath, + MAX_INPUT_BYTES, + 'Autosave benchmark input must be a regular non-symlink file.', + 'Autosave benchmark input exceeds the supported size.', + ); + let envelope; + try { + envelope = JSON.parse(source.toString('utf8')); + } catch { + throw new Error('Autosave benchmark input must be a valid document envelope.'); + } + if ( + envelope?.schemaId !== + 'https://inkspan.io/schemas/document-envelope/v1' || + envelope.schemaVersion !== 1 || + typeof envelope.documentJson !== 'object' || + envelope.documentJson === null + ) { + throw new Error('Autosave benchmark input must be a valid document envelope.'); + } + const digestHex = createHash('sha256').update(source).digest('hex'); + return Object.freeze({ + envelope, + revision: Object.freeze({ + algorithm: 'SHA-256', + digestHex, + strongEntityTag: `"sha256-${digestHex}"`, + }), + }); + } const textNode = Object.freeze({ type: 'text', text: SYNTHETIC_DOCUMENT_LABEL }); const paragraph = Object.freeze({ type: 'paragraph', @@ -391,7 +429,7 @@ async function main() { 'Measured autosave module must export createDocumentAutosaveQueue().', ); } - const evidence = createSyntheticRevisionEvidence(); + const evidence = createSyntheticRevisionEvidence(args.inputPath); await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence); const samples = []; diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index fee54756..b48ca5ec 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -117,6 +117,8 @@ function measurementArguments({ function autosaveMeasurementArguments({ modulePath, artifactSha256, shared }) { return Object.freeze([ + '--input', + shared.revisionInputPath, '--module', modulePath, '--profile', diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index d63ae39a..71c0289e 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -35,11 +35,13 @@ describe('autosave enqueue performance measurement', () => { it('emits the canonical summarizable sample contract without persisting document content', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-measure-')); const modulePath = join(directory, 'autosave.mjs'); + const inputPath = join(directory, 'document-envelope.json'); const outputPath = join(directory, 'samples.json'); const moduleSource = [ 'export function createDocumentAutosaveQueue(options) {', ' return Object.freeze({', ' async enqueue(evidence) {', + " if (evidence.envelope.documentJson.content[0].content[0].text !== 'profile-bound synthetic input') throw new Error('wrong profile input');", ' const result = await options.save(evidence);', " if (result?.status !== 'saved') throw new Error('save failed');", ' return Object.freeze({', @@ -58,11 +60,18 @@ describe('autosave enqueue performance measurement', () => { try { writeFileSync(modulePath, moduleSource, 'utf8'); + writeFileSync( + inputPath, + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"profile-bound synthetic input"}]}]}}\n', + 'utf8', + ); const result = spawnSync( process.execPath, [ measurementScript, + '--input', + inputPath, '--module', modulePath, '--profile', diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 6c1aa315..aba6923d 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -166,7 +166,7 @@ describe('packed artifact benchmark path stability', () => { writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); writeFileSync( revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Stable packed artifact"}\n', + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Stable packed artifact"}]}]}}\n', 'utf8', ); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 97dc8ca6..1fee372d 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -123,7 +123,7 @@ function packedSuiteArguments(options: { writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); writeFileSync( revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Packed buyer benchmark"}]}]}}\n', 'utf8', ); return [ From d0e353a5855c2e21009e0f35f8a92e6fad7c5dc1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:43:59 +0900 Subject: [PATCH 232/260] fix(perf): measure autosave with profile input Signed-off-by: Seongho Bae --- benchmarks/measure-autosave.mjs | 60 +++++++++++++++++-- benchmarks/run-current-suite-core.mjs | 16 ++++- ...ormanceAutosaveMeasurementContract.test.ts | 25 ++++++++ ...ormancePackedArtifactPathStability.test.ts | 2 +- ...ormancePackedArtifactSuiteContract.test.ts | 2 +- 5 files changed, 98 insertions(+), 7 deletions(-) diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 055e7e81..163b8b16 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -19,6 +19,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); @@ -35,7 +36,7 @@ const OUTPUT_EXISTS_ERROR = 'Autosave benchmark output must not already exist.'; const SYNTHETIC_DOCUMENT_LABEL = 'Synthetic autosave benchmark document'; function resolveArguments(argv) { - const expectedFlags = [ + const legacyFlags = [ '--module', '--profile', '--samples', @@ -45,13 +46,21 @@ function resolveArguments(argv) { '--reference-hardware-id', '--output', ]; + const inputFlags = [ + '--input', + '--revision-module', + '--revision-artifact-sha256', + ...legacyFlags, + ]; + const expectedFlags = + argv.length === inputFlags.length * 2 ? inputFlags : legacyFlags; if ( argv.length !== expectedFlags.length * 2 || expectedFlags.some((flag, index) => argv[index * 2] !== flag) || expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) ) { throw new Error( - 'Usage: node benchmarks/measure-autosave.mjs --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + 'Usage: node benchmarks/measure-autosave.mjs [--input --revision-module --revision-artifact-sha256 ] --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } @@ -84,6 +93,15 @@ function resolveArguments(argv) { 'Autosave benchmark artifact digest must be a lowercase 64-character SHA-256.', ); } + const revisionArtifactSha256 = values['--revision-artifact-sha256']; + if ( + values['--input'] !== undefined && + !SHA256_PATTERN.test(revisionArtifactSha256) + ) { + throw new Error( + 'Autosave benchmark revision artifact digest must be a lowercase 64-character SHA-256.', + ); + } const runtimeId = values['--runtime-id']; if (!RUNTIME_ID_PATTERN.test(runtimeId)) { throw new Error('Autosave benchmark runtime ID is invalid.'); @@ -94,6 +112,10 @@ function resolveArguments(argv) { } return Object.freeze({ + inputPath: + values['--input'] === undefined ? null : resolve(values['--input']), + revisionModulePath: values['--revision-module'], + revisionArtifactSha256, modulePath: values['--module'], profile, sampleCount, @@ -294,7 +316,37 @@ async function loadMeasuredModule(modulePath) { } } -function createSyntheticRevisionEvidence() { +async function createSyntheticRevisionEvidence(args) { + const inputPath = args.inputPath; + if (inputPath !== null) { + const source = readBoundedRegularFile( + inputPath, + MAX_INPUT_BYTES, + 'Autosave benchmark input must be a regular non-symlink file.', + 'Autosave benchmark input exceeds the supported size.', + ); + const revisionModulePath = resolveLocalModule(args.revisionModulePath); + verifyMeasuredModuleDigest( + revisionModulePath, + args.revisionArtifactSha256, + ); + const revisionModule = await loadMeasuredModule(revisionModulePath); + if ( + typeof revisionModule.createDocumentEnvelopeRevisionEvidenceBytes !== + 'function' + ) { + throw new Error( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + } + try { + return await revisionModule.createDocumentEnvelopeRevisionEvidenceBytes( + source, + ); + } catch { + throw new Error('Autosave benchmark input must be a valid document envelope.'); + } + } const textNode = Object.freeze({ type: 'text', text: SYNTHETIC_DOCUMENT_LABEL }); const paragraph = Object.freeze({ type: 'paragraph', @@ -391,7 +443,7 @@ async function main() { 'Measured autosave module must export createDocumentAutosaveQueue().', ); } - const evidence = createSyntheticRevisionEvidence(); + const evidence = await createSyntheticRevisionEvidence(args); await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence); const samples = []; diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index fee54756..108f2ddb 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -115,8 +115,20 @@ function measurementArguments({ ]); } -function autosaveMeasurementArguments({ modulePath, artifactSha256, shared }) { +function autosaveMeasurementArguments({ + modulePath, + artifactSha256, + revisionModulePath, + revisionArtifactSha256, + shared, +}) { return Object.freeze([ + '--input', + shared.revisionInputPath, + '--revision-module', + revisionModulePath, + '--revision-artifact-sha256', + revisionArtifactSha256, '--module', modulePath, '--profile', @@ -643,6 +655,8 @@ function main(argv) { ? autosaveMeasurementArguments({ modulePath: preparedPackage.autosaveModulePath, artifactSha256: preparedPackage.autosaveArtifactSha256, + revisionModulePath: preparedPackage.revisionModulePath, + revisionArtifactSha256: preparedPackage.revisionArtifactSha256, shared, }) : null; diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index d63ae39a..48d7bc55 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -35,11 +35,14 @@ describe('autosave enqueue performance measurement', () => { it('emits the canonical summarizable sample contract without persisting document content', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-measure-')); const modulePath = join(directory, 'autosave.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const inputPath = join(directory, 'document-envelope.json'); const outputPath = join(directory, 'samples.json'); const moduleSource = [ 'export function createDocumentAutosaveQueue(options) {', ' return Object.freeze({', ' async enqueue(evidence) {', + " if (evidence.envelope.documentJson.content[0].content[0].text !== 'profile-bound synthetic input') throw new Error('wrong profile input');", ' const result = await options.save(evidence);', " if (result?.status !== 'saved') throw new Error('save failed');", ' return Object.freeze({', @@ -55,14 +58,36 @@ describe('autosave enqueue performance measurement', () => { '}', '', ].join('\n'); + const revisionModuleSource = [ + "import { createHash } from 'node:crypto';", + 'function freeze(value) { if (value && typeof value === \'object\') { for (const child of Object.values(value)) freeze(child); Object.freeze(value); } return value; }', + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " const envelope = freeze(JSON.parse(Buffer.from(source).toString('utf8')));", + " const digestHex = createHash('sha256').update(source).digest('hex');", + ' return Object.freeze({ envelope, revision: Object.freeze({ algorithm: \'SHA-256\', digestHex, strongEntityTag: `"sha256-${digestHex}"` }) });', + '}', + '', + ].join('\n'); try { writeFileSync(modulePath, moduleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + writeFileSync( + inputPath, + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"profile-bound synthetic input"}]}]}}\n', + 'utf8', + ); const result = spawnSync( process.execPath, [ measurementScript, + '--input', + inputPath, + '--revision-module', + revisionModulePath, + '--revision-artifact-sha256', + sha256(revisionModuleSource), '--module', modulePath, '--profile', diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 6c1aa315..aba6923d 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -166,7 +166,7 @@ describe('packed artifact benchmark path stability', () => { writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); writeFileSync( revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Stable packed artifact"}\n', + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Stable packed artifact"}]}]}}\n', 'utf8', ); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 97dc8ca6..1fee372d 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -123,7 +123,7 @@ function packedSuiteArguments(options: { writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); writeFileSync( revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Packed buyer benchmark"}]}]}}\n', 'utf8', ); return [ From b434cc85fb4013ffbacbdc787c9b461817795d4e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:48:28 +0900 Subject: [PATCH 233/260] fix(perf): scale envelope structure by profile Signed-off-by: Seongho Bae --- benchmarks/corpus.lock.json | 16 ++++++++-------- benchmarks/generate-corpus.mjs | 7 ++++++- src/performanceCorpusContract.test.ts | 8 +++++--- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index c0c834b8..5ee1b96c 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -14,29 +14,29 @@ "sections": 1, "bytes": 2152, "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea", - "envelopeBytes": 2374, - "envelopeSha256": "0bb4a9ce44d3f93713fec4ef636177bf9693670ba59434fb818b461e8546035a" + "envelopeBytes": 4236, + "envelopeSha256": "7ee1425e1f57808a3a4d11987a0a6bc2aec97d311457f5456e82f1adbc48c810" }, "medium": { "sections": 8, "bytes": 15712, "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c", - "envelopeBytes": 16186, - "envelopeSha256": "240c22c05b8889e3a8e9f211c18227053955d2146c43a25cb010cfee975d6486" + "envelopeBytes": 30018, + "envelopeSha256": "47fa0cd11ffec5f9c5513a935180d1d7249157c46a0e115d1de938be86fa4459" }, "large": { "sections": 32, "bytes": 62199, "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392", - "envelopeBytes": 63537, - "envelopeSha256": "e04c53670213b4537243ba8b7c72036b5f0837f40cd150f6947ec38bf9e1b0d4" + "envelopeBytes": 118409, + "envelopeSha256": "c93143659a352cffaa1ff817b0efa0ba4b63f8410bbb7d923799b827a36aa655" }, "stress": { "sections": 128, "bytes": 248152, "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928", - "envelopeBytes": 252946, - "envelopeSha256": "858e857576c95e1259b3a817412f29b916fad8df0aa6b104cb39ecc473a5719c" + "envelopeBytes": 471978, + "envelopeSha256": "ebc47c89671e39af349a4cb65ba15ed3138a3eee187245e520abe80d877bcd6c" } } } diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index 53bba374..f8011aac 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -135,12 +135,17 @@ function buildProfile(profile, sectionCount) { } function buildEnvelope(body) { + const content = body.split('\n').map((line) => + line.length === 0 + ? { type: 'paragraph' } + : { type: 'paragraph', content: [{ type: 'text', text: line }] }, + ); return `${JSON.stringify({ schemaId: 'https://inkspan.io/schemas/document-envelope/v1', schemaVersion: 1, documentJson: { type: 'doc', - content: [{ type: 'paragraph', content: [{ type: 'text', text: body }] }], + content, }, })}\n`; } diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 0a5df7e7..e059e59b 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -90,9 +90,11 @@ describe('deterministic synthetic performance corpus', () => { expected.profiles[profile].envelopeBytes, ); expect( - JSON.parse(firstEnvelope.toString('utf8')).documentJson.content[0] - .content[0].text, - ).toBe(firstBytes.toString('utf8')); + JSON.parse(firstEnvelope.toString('utf8')).documentJson.content.map( + (node: { content?: readonly [{ text: string }] }) => + node.content?.[0].text ?? '', + ), + ).toEqual(firstBytes.toString('utf8').split('\n')); } const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); From 466326ff0c89cb40ff9a60fef95b994734dec18f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:11:26 +0900 Subject: [PATCH 234/260] perf: measure packed transition evidence Signed-off-by: Seongho Bae --- benchmarks/measure-revision-evidence.mjs | 67 +++++++++++++------ benchmarks/run-current-suite-core.mjs | 21 ++++++ ...anceHtmlSerializationSuiteContract.test.ts | 5 +- ...ormancePackedArtifactPathStability.test.ts | 5 +- ...ormancePackedArtifactSuiteContract.test.ts | 5 +- ...ormanceRevisionMeasurementContract.test.ts | 32 +++++++++ ...formanceSingleCommandSuiteContract.test.ts | 14 +++- 7 files changed, 123 insertions(+), 26 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 1e3e438d..30e42c5a 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -36,7 +36,7 @@ const OUTPUT_EXISTS_ERROR = 'Revision benchmark output must not already exist.'; function resolveArguments(argv) { - const expectedFlags = [ + const baseFlags = [ '--input', '--module', '--profile', @@ -47,6 +47,9 @@ function resolveArguments(argv) { '--reference-hardware-id', '--output', ]; + const operationFlags = [...baseFlags.slice(0, -1), '--operation', '--output']; + const expectedFlags = + argv.length === operationFlags.length * 2 ? operationFlags : baseFlags; if ( argv.length !== expectedFlags.length * 2 || expectedFlags.some((flag, index) => argv[index * 2] !== flag) || @@ -92,6 +95,10 @@ function resolveArguments(argv) { if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { throw new Error('Revision benchmark reference hardware ID is invalid.'); } + const operation = values['--operation'] ?? 'revision'; + if (operation !== 'revision' && operation !== 'transition') { + throw new Error('Revision benchmark operation is invalid.'); + } return Object.freeze({ inputPath: resolve(values['--input']), @@ -102,6 +109,7 @@ function resolveArguments(argv) { artifactSha256, runtimeId, referenceHardwareId, + operation, outputPath: resolve(values['--output']), }); } @@ -327,29 +335,45 @@ function writeMeasurementOutput(path, content) { } } -async function runMeasuredRevision(createRevisionEvidence, source) { +async function runMeasuredEvidence(createEvidence, source, operation) { let evidence; try { - evidence = await createRevisionEvidence(source); + evidence = + operation === 'revision' + ? await createEvidence(source) + : await createEvidence(source, source); } catch { throw new Error('Measured revision-evidence execution failed.'); } - let digestHex; + let revisions; try { if (typeof evidence !== 'object' || evidence === null) { throw new Error('invalid revision evidence'); } - const revision = evidence.revision; - if (typeof revision !== 'object' || revision === null) { + revisions = + operation === 'revision' + ? [evidence.revision] + : [evidence.previousRevision, evidence.resultingRevision]; + if ( + (operation === 'transition' && typeof evidence.changed !== 'boolean') || + revisions.some( + (revision) => typeof revision !== 'object' || revision === null, + ) + ) { throw new Error('invalid revision evidence'); } - digestHex = revision.digestHex; } catch { throw new Error('Measured revision-evidence result is invalid.'); } - if (typeof digestHex !== 'string' || !SHA256_PATTERN.test(digestHex)) { + if ( + revisions.some( + (revision) => + typeof revision.digestHex !== 'string' || + !SHA256_PATTERN.test(revision.digestHex), + ) + ) { throw new Error('Measured revision-evidence result is invalid.'); } } @@ -377,27 +401,26 @@ async function main() { assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); - if ( - typeof measuredModule.createDocumentEnvelopeRevisionEvidenceBytes !== - 'function' - ) { + const createEvidence = + args.operation === 'revision' + ? measuredModule.createDocumentEnvelopeRevisionEvidenceBytes + : measuredModule.createDocumentEnvelopeTransitionEvidenceBytes; + if (typeof createEvidence !== 'function') { throw new Error( - 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + `Measured revision module must export ${ + args.operation === 'revision' + ? 'createDocumentEnvelopeRevisionEvidenceBytes' + : 'createDocumentEnvelopeTransitionEvidenceBytes' + }().`, ); } - await runMeasuredRevision( - measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, - source, - ); + await runMeasuredEvidence(createEvidence, source, args.operation); const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - await runMeasuredRevision( - measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, - source, - ); + await runMeasuredEvidence(createEvidence, source, args.operation); const elapsed = performance.now() - start; if (!Number.isFinite(elapsed) || elapsed < 0) { throw new Error('Revision measurement produced invalid runtime evidence.'); @@ -423,7 +446,7 @@ async function main() { `${JSON.stringify( { contractVersion: 1, - benchmarkId: `revision-evidence-${args.profile}`, + benchmarkId: `${args.operation}-evidence-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index 108f2ddb..d0abc94a 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -558,6 +558,16 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) 'revision', 'summary', ); + const transitionSamplesPath = resolve( + args.outputDirectory, + 'transition', + 'samples.json', + ); + const transitionSummaryDirectory = resolve( + args.outputDirectory, + 'transition', + 'summary', + ); runMeasurementAndSummary({ measurementScript: 'measure-markdown.mjs', @@ -575,6 +585,14 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) measurementFailure: 'Benchmark suite revision measurement failed.', summaryFailure: 'Benchmark suite revision summary failed.', }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [...revisionArguments, '--operation', 'transition'], + samplesPath: transitionSamplesPath, + summaryDirectory: transitionSummaryDirectory, + measurementFailure: 'Benchmark suite transition measurement failed.', + summaryFailure: 'Benchmark suite transition summary failed.', + }); if (autosaveArguments !== null) { const autosaveSamplesPath = resolve( @@ -613,6 +631,9 @@ function suiteManifest(args, packageEvidence, includeAutosave) { revisionSamples: 'revision/samples.json', revisionSummaryJson: 'revision/summary/summary.json', revisionSummaryText: 'revision/summary/summary.txt', + transitionSamples: 'transition/samples.json', + transitionSummaryJson: 'transition/summary/summary.json', + transitionSummaryText: 'transition/summary/summary.txt', ...(includeAutosave ? { autosaveSamples: 'autosave/samples.json', diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index 729acdc8..8679dae8 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -41,7 +41,10 @@ describe('single-command HTML serialization benchmark contract', () => { "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", '', ].join('\n'); - const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + const revisionModuleSource = `const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`; try { writeFileSync(markdownInput, '# Buyer benchmark\n', 'utf8'); diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index aba6923d..26a15671 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -72,7 +72,10 @@ function createPackedBenchmarkFixture( ); writeFileSync( join(distDirectory, 'cwl-revision-evidence.js'), - `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'e'.repeat(64)}' } }; }\n`, + `const revision = { digestHex: '${'e'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`, 'utf8', ); writeFileSync( diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 1fee372d..415481d4 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -68,7 +68,10 @@ function createPackedBenchmarkFixture(directory: string): { ); writeFileSync( join(distDirectory, 'cwl-revision-evidence.js'), - `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`, + `const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`, 'utf8', ); writeFileSync( diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 8df70188..92fd6d67 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -49,6 +49,7 @@ function argumentsFor( input: string, modulePath: string, output: string, + operation?: 'transition', ): string[] { return [ measurementScript, @@ -68,6 +69,7 @@ function argumentsFor( RUNTIME_ID, '--reference-hardware-id', HARDWARE_ID, + ...(operation === undefined ? [] : ['--operation', operation]), '--output', output, ]; @@ -153,6 +155,36 @@ describe('revision-evidence runtime measurement contract', () => { } }); + it('measures transition evidence through the same packed module boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `const revision = { digestHex: '${'d'.repeat(64)}' }; +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, + 'utf8', + ); + + execFileSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'transition'), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ + benchmarkId: 'transition-evidence-large', + artifactSha256: fileSha256(modulePath), + samples: expect.any(Array), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed before output when the measured module lacks the revision API', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); const input = join(root, 'small.json'); diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 83490f29..7f268173 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -87,7 +87,10 @@ function writeBenchmarkInputs(directory: string): { "export function markdownToHtml(source) { return `

${source}

`; }\n"; const revisionInputPath = join(directory, 'document-envelope.json'); const revisionModulePath = join(directory, 'revision-measured.mjs'); - const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + const revisionModuleSource = `const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`; writeFileSync(markdownInputPath, '# Buyer benchmark\n', 'utf8'); writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); @@ -151,6 +154,9 @@ describe('single-command benchmark suite contract', () => { revisionSamples: 'revision/samples.json', revisionSummaryJson: 'revision/summary/summary.json', revisionSummaryText: 'revision/summary/summary.txt', + transitionSamples: 'transition/samples.json', + transitionSummaryJson: 'transition/summary/summary.json', + transitionSummaryText: 'transition/summary/summary.txt', status: 'completed', }); @@ -197,6 +203,12 @@ describe('single-command benchmark suite contract', () => { 'utf8', ), ).toContain('revision-evidence-small'); + + const transitionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'transition', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; samples?: unknown }; + expect(transitionSamples.benchmarkId).toBe('transition-evidence-small'); + expect(transitionSamples.samples).toHaveLength(2); }, 20_000); it('rejects a claimed source commit that is not the checked-out HEAD', () => { From d36227a93f45ff7c910927cba7ca3bdfb5a7570f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:15:11 +0900 Subject: [PATCH 235/260] fix(perf): redact transition result inspection Signed-off-by: Seongho Bae --- benchmarks/measure-revision-evidence.mjs | 14 +++------ ...ormanceRevisionMeasurementContract.test.ts | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 30e42c5a..35706f6e 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -359,6 +359,11 @@ async function runMeasuredEvidence(createEvidence, source, operation) { (operation === 'transition' && typeof evidence.changed !== 'boolean') || revisions.some( (revision) => typeof revision !== 'object' || revision === null, + ) || + revisions.some( + (revision) => + typeof revision.digestHex !== 'string' || + !SHA256_PATTERN.test(revision.digestHex), ) ) { throw new Error('invalid revision evidence'); @@ -367,15 +372,6 @@ async function runMeasuredEvidence(createEvidence, source, operation) { throw new Error('Measured revision-evidence result is invalid.'); } - if ( - revisions.some( - (revision) => - typeof revision.digestHex !== 'string' || - !SHA256_PATTERN.test(revision.digestHex), - ) - ) { - throw new Error('Measured revision-evidence result is invalid.'); - } } async function main() { diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 92fd6d67..324714d8 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -243,6 +243,37 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { } }); + it('redacts hostile transition-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-transition-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeTransitionEvidenceBytes() { const revision = new Proxy({}, { get() { throw new Error('${privateSentinel}'); } }); return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'transition'), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('redacts filesystem details when output path traversal fails', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-output-')); const input = join(root, 'large.json'); From f7da6b39c6e1254d5528eff765512e1e0099b70a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:20:28 +0900 Subject: [PATCH 236/260] perf: measure autosave coalescing Signed-off-by: Seongho Bae --- benchmarks/measure-autosave.mjs | 66 +++++++++++++++++-- benchmarks/run-current-suite-core.mjs | 23 +++++++ ...ormanceAutosaveMeasurementContract.test.ts | 45 +++++++++++++ ...ormancePackedArtifactPathStability.test.ts | 6 +- ...ormancePackedArtifactSuiteContract.test.ts | 20 +++++- 5 files changed, 154 insertions(+), 6 deletions(-) diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 163b8b16..3e4d69a2 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -52,8 +52,20 @@ function resolveArguments(argv) { '--revision-artifact-sha256', ...legacyFlags, ]; + const operationFlags = [...legacyFlags.slice(0, -1), '--operation', '--output']; + const inputOperationFlags = [ + ...inputFlags.slice(0, -1), + '--operation', + '--output', + ]; const expectedFlags = - argv.length === inputFlags.length * 2 ? inputFlags : legacyFlags; + argv.length === inputOperationFlags.length * 2 + ? inputOperationFlags + : argv.length === inputFlags.length * 2 + ? inputFlags + : argv.length === operationFlags.length * 2 + ? operationFlags + : legacyFlags; if ( argv.length !== expectedFlags.length * 2 || expectedFlags.some((flag, index) => argv[index * 2] !== flag) || @@ -110,6 +122,10 @@ function resolveArguments(argv) { if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { throw new Error('Autosave benchmark reference hardware ID is invalid.'); } + const operation = values['--operation'] ?? 'enqueue'; + if (operation !== 'enqueue' && operation !== 'coalescing') { + throw new Error('Autosave benchmark operation is invalid.'); + } return Object.freeze({ inputPath: @@ -123,6 +139,7 @@ function resolveArguments(argv) { artifactSha256, runtimeId, referenceHardwareId, + operation, outputPath: resolve(values['--output']), }); } @@ -407,6 +424,45 @@ async function measureOneEnqueue(createDocumentAutosaveQueue, evidence) { return elapsed; } +async function measureOneCoalescing(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + let releaseSave; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return new Promise((resolve) => { + releaseSave = () => resolve(Object.freeze({ status: 'saved' })); + }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + const active = queue.enqueue(evidence); + await Promise.resolve(); + if (typeof releaseSave !== 'function') { + throw new Error('Measured autosave coalescing setup is invalid.'); + } + + const start = performance.now(); + const coalesced = queue.enqueue(evidence); + const elapsed = performance.now() - start; + if (coalesced !== active || saveCalls !== 1) { + throw new Error('Measured autosave coalescing result is invalid.'); + } + releaseSave(); + const outcome = await active; + if (outcome?.status !== 'saved') { + throw new Error('Measured autosave coalescing result is invalid.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + function writeMeasurementOutput(path, content) { assertNoSymlinkOutputAncestors(path); try { @@ -445,11 +501,13 @@ async function main() { } const evidence = await createSyntheticRevisionEvidence(args); - await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence); + const measure = + args.operation === 'enqueue' ? measureOneEnqueue : measureOneCoalescing; + await measure(measuredModule.createDocumentAutosaveQueue, evidence); const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { samples.push( - await measureOneEnqueue(measuredModule.createDocumentAutosaveQueue, evidence), + await measure(measuredModule.createDocumentAutosaveQueue, evidence), ); } @@ -470,7 +528,7 @@ async function main() { `${JSON.stringify( { contractVersion: 1, - benchmarkId: `autosave-enqueue-${args.profile}`, + benchmarkId: `autosave-${args.operation}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index d0abc94a..4bd419b7 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -605,6 +605,16 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) 'autosave', 'summary', ); + const autosaveCoalescingSamplesPath = resolve( + args.outputDirectory, + 'autosave-coalescing', + 'samples.json', + ); + const autosaveCoalescingSummaryDirectory = resolve( + args.outputDirectory, + 'autosave-coalescing', + 'summary', + ); runMeasurementAndSummary({ measurementScript: 'measure-autosave.mjs', measurementArguments: autosaveArguments, @@ -613,6 +623,14 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) measurementFailure: 'Benchmark suite autosave measurement failed.', summaryFailure: 'Benchmark suite autosave summary failed.', }); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: [...autosaveArguments, '--operation', 'coalescing'], + samplesPath: autosaveCoalescingSamplesPath, + summaryDirectory: autosaveCoalescingSummaryDirectory, + measurementFailure: 'Benchmark suite autosave coalescing measurement failed.', + summaryFailure: 'Benchmark suite autosave coalescing summary failed.', + }); } } @@ -639,6 +657,11 @@ function suiteManifest(args, packageEvidence, includeAutosave) { autosaveSamples: 'autosave/samples.json', autosaveSummaryJson: 'autosave/summary/summary.json', autosaveSummaryText: 'autosave/summary/summary.txt', + autosaveCoalescingSamples: 'autosave-coalescing/samples.json', + autosaveCoalescingSummaryJson: + 'autosave-coalescing/summary/summary.json', + autosaveCoalescingSummaryText: + 'autosave-coalescing/summary/summary.txt', } : {}), status: 'completed', diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index 48d7bc55..4af0e653 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -164,4 +164,49 @@ describe('autosave enqueue performance measurement', () => { rmSync(directory, { recursive: true, force: true }); } }, 20_000); + + it('measures active-revision coalescing without starting a second save', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-coalescing-')); + const modulePath = join(directory, 'autosave.mjs'); + const outputPath = join(directory, 'samples.json'); + const moduleSource = `export function createDocumentAutosaveQueue(options) { + let active; + return { + enqueue(evidence) { + if (active) return active; + active = Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); + return active; + }, + async close() {}, + }; +}\n`; + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + const args = [ + measurementScript, + '--module', modulePath, + '--profile', 'small', + '--samples', '2', + '--source-commit-sha', sourceCommitSha, + '--artifact-sha256', sha256(moduleSource), + '--runtime-id', runtimeId, + '--reference-hardware-id', referenceHardwareId, + '--operation', 'coalescing', + '--output', outputPath, + ]; + + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toMatchObject({ + benchmarkId: 'autosave-coalescing-small', + samples: [expect.any(Number), expect.any(Number)], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); }); diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 26a15671..f851ec07 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -82,8 +82,12 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { join(distDirectory, 'cwl-autosave.js'), [ 'export function createDocumentAutosaveQueue({ save }) {', + ' let active;', ' return {', - ' async enqueue(evidence) { return await save(evidence); },', + ' enqueue(evidence) {', + " active ??= Promise.resolve(save(evidence)).then(() => ({ status: 'saved' }));", + ' return active;', + ' },', ' async close() {},', ' };', '}', diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 415481d4..784e33b9 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -78,8 +78,12 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { join(distDirectory, 'cwl-autosave.js'), [ 'export function createDocumentAutosaveQueue({ save }) {', + ' let active;', ' return {', - ' async enqueue(evidence) { return await save(evidence); },', + ' enqueue(evidence) {', + " active ??= Promise.resolve(save(evidence)).then(() => ({ status: 'saved' }));", + ' return active;', + ' },', ' async close() {},', ' };', '}', @@ -193,6 +197,11 @@ describe('packed artifact benchmark suite contract', () => { autosaveSamples: 'autosave/samples.json', autosaveSummaryJson: 'autosave/summary/summary.json', autosaveSummaryText: 'autosave/summary/summary.txt', + autosaveCoalescingSamples: 'autosave-coalescing/samples.json', + autosaveCoalescingSummaryJson: + 'autosave-coalescing/summary/summary.json', + autosaveCoalescingSummaryText: + 'autosave-coalescing/summary/summary.txt', htmlSerializationSamples: 'html-serialization/samples.json', htmlSerializationSummaryJson: 'html-serialization/summary/summary.json', @@ -210,6 +219,15 @@ describe('packed artifact benchmark suite contract', () => { expect(autosaveSamples.benchmarkId).toBe('autosave-enqueue-small'); expect(autosaveSamples.samples).toHaveLength(2); + const coalescingSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave-coalescing', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(coalescingSamples.benchmarkId).toBe('autosave-coalescing-small'); + expect(coalescingSamples.samples).toHaveLength(2); + const htmlSamples = JSON.parse( readFileSync( join(directory, 'evidence', 'html-serialization', 'samples.json'), From e22141c3a408943ed41739d1ec85b4f66958292e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:26:47 +0900 Subject: [PATCH 237/260] perf: measure autosave commit classification Signed-off-by: Seongho Bae --- benchmarks/measure-autosave.mjs | 51 +++++++++++++++++-- benchmarks/run-current-suite-core.mjs | 21 ++++++++ ...ormanceAutosaveMeasurementContract.test.ts | 40 +++++++++++++++ ...ormancePackedArtifactSuiteContract.test.ts | 12 +++++ 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 3e4d69a2..e2ca1295 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -123,7 +123,11 @@ function resolveArguments(argv) { throw new Error('Autosave benchmark reference hardware ID is invalid.'); } const operation = values['--operation'] ?? 'enqueue'; - if (operation !== 'enqueue' && operation !== 'coalescing') { + if ( + operation !== 'enqueue' && + operation !== 'coalescing' && + operation !== 'commit' + ) { throw new Error('Autosave benchmark operation is invalid.'); } @@ -463,6 +467,44 @@ async function measureOneCoalescing(createDocumentAutosaveQueue, evidence) { return elapsed; } +async function measureOneCommit(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + let releaseSave; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return new Promise((resolve) => { + releaseSave = () => resolve(Object.freeze({ status: 'saved' })); + }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + const active = queue.enqueue(evidence); + await Promise.resolve(); + if (typeof releaseSave !== 'function' || saveCalls !== 1) { + throw new Error('Measured autosave commit setup is invalid.'); + } + + const start = performance.now(); + releaseSave(); + const outcome = await active; + const elapsed = performance.now() - start; + if (outcome?.status !== 'saved') { + throw new Error('Measured autosave commit result is invalid.'); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Autosave measurement produced invalid runtime evidence.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + function writeMeasurementOutput(path, content) { assertNoSymlinkOutputAncestors(path); try { @@ -501,8 +543,11 @@ async function main() { } const evidence = await createSyntheticRevisionEvidence(args); - const measure = - args.operation === 'enqueue' ? measureOneEnqueue : measureOneCoalescing; + const measure = { + enqueue: measureOneEnqueue, + coalescing: measureOneCoalescing, + commit: measureOneCommit, + }[args.operation]; await measure(measuredModule.createDocumentAutosaveQueue, evidence); const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index 4bd419b7..7803ec4a 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -615,6 +615,16 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) 'autosave-coalescing', 'summary', ); + const autosaveCommitSamplesPath = resolve( + args.outputDirectory, + 'autosave-commit', + 'samples.json', + ); + const autosaveCommitSummaryDirectory = resolve( + args.outputDirectory, + 'autosave-commit', + 'summary', + ); runMeasurementAndSummary({ measurementScript: 'measure-autosave.mjs', measurementArguments: autosaveArguments, @@ -631,6 +641,14 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) measurementFailure: 'Benchmark suite autosave coalescing measurement failed.', summaryFailure: 'Benchmark suite autosave coalescing summary failed.', }); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: [...autosaveArguments, '--operation', 'commit'], + samplesPath: autosaveCommitSamplesPath, + summaryDirectory: autosaveCommitSummaryDirectory, + measurementFailure: 'Benchmark suite autosave commit measurement failed.', + summaryFailure: 'Benchmark suite autosave commit summary failed.', + }); } } @@ -662,6 +680,9 @@ function suiteManifest(args, packageEvidence, includeAutosave) { 'autosave-coalescing/summary/summary.json', autosaveCoalescingSummaryText: 'autosave-coalescing/summary/summary.txt', + autosaveCommitSamples: 'autosave-commit/samples.json', + autosaveCommitSummaryJson: 'autosave-commit/summary/summary.json', + autosaveCommitSummaryText: 'autosave-commit/summary/summary.txt', } : {}), status: 'completed', diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index 4af0e653..fca9984e 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -209,4 +209,44 @@ describe('autosave enqueue performance measurement', () => { rmSync(directory, { recursive: true, force: true }); } }); + + it('measures saved-result classification after the host save completes', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-commit-')); + const modulePath = join(directory, 'autosave.mjs'); + const outputPath = join(directory, 'samples.json'); + const moduleSource = `export function createDocumentAutosaveQueue(options) { + return { + enqueue(evidence) { + return Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); + }, + async close() {}, + }; +}\n`; + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync(process.execPath, [ + measurementScript, + '--module', modulePath, + '--profile', 'small', + '--samples', '2', + '--source-commit-sha', sourceCommitSha, + '--artifact-sha256', sha256(moduleSource), + '--runtime-id', runtimeId, + '--reference-hardware-id', referenceHardwareId, + '--operation', 'commit', + '--output', outputPath, + ], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toMatchObject({ + benchmarkId: 'autosave-commit-small', + samples: [expect.any(Number), expect.any(Number)], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); }); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 784e33b9..b35aa1bd 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -202,6 +202,9 @@ describe('packed artifact benchmark suite contract', () => { 'autosave-coalescing/summary/summary.json', autosaveCoalescingSummaryText: 'autosave-coalescing/summary/summary.txt', + autosaveCommitSamples: 'autosave-commit/samples.json', + autosaveCommitSummaryJson: 'autosave-commit/summary/summary.json', + autosaveCommitSummaryText: 'autosave-commit/summary/summary.txt', htmlSerializationSamples: 'html-serialization/samples.json', htmlSerializationSummaryJson: 'html-serialization/summary/summary.json', @@ -228,6 +231,15 @@ describe('packed artifact benchmark suite contract', () => { expect(coalescingSamples.benchmarkId).toBe('autosave-coalescing-small'); expect(coalescingSamples.samples).toHaveLength(2); + const commitSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave-commit', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(commitSamples.benchmarkId).toBe('autosave-commit-small'); + expect(commitSamples.samples).toHaveLength(2); + const htmlSamples = JSON.parse( readFileSync( join(directory, 'evidence', 'html-serialization', 'samples.json'), From f6875f4c735d86bbf548820e7e97d4183d83521d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:32:33 +0900 Subject: [PATCH 238/260] perf: measure envelope canonicalization Signed-off-by: Seongho Bae --- benchmarks/measure-revision-evidence.mjs | 56 ++++++++++++++----- benchmarks/run-current-suite-core.mjs | 28 +++++++++- ...ormancePackedArtifactPathStability.test.ts | 6 +- ...ormancePackedArtifactSuiteContract.test.ts | 22 +++++++- ...ormanceRevisionMeasurementContract.test.ts | 34 ++++++++++- 5 files changed, 128 insertions(+), 18 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 35706f6e..4eb32ba4 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -96,7 +96,11 @@ function resolveArguments(argv) { throw new Error('Revision benchmark reference hardware ID is invalid.'); } const operation = values['--operation'] ?? 'revision'; - if (operation !== 'revision' && operation !== 'transition') { + if ( + operation !== 'revision' && + operation !== 'transition' && + operation !== 'canonicalization' + ) { throw new Error('Revision benchmark operation is invalid.'); } @@ -336,12 +340,32 @@ function writeMeasurementOutput(path, content) { } async function runMeasuredEvidence(createEvidence, source, operation) { + let digestCalls = 0; + const canonicalizationDigestProvider = { + digest(algorithm, canonicalSource) { + digestCalls += 1; + if ( + algorithm !== 'SHA-256' || + !ArrayBuffer.isView(canonicalSource) || + canonicalSource.byteLength === 0 + ) { + throw new Error('invalid canonical source'); + } + return Promise.resolve(new Uint8Array(32).buffer); + }, + }; let evidence; try { evidence = - operation === 'revision' - ? await createEvidence(source) - : await createEvidence(source, source); + operation === 'transition' + ? await createEvidence(source, source) + : await createEvidence( + source, + undefined, + operation === 'canonicalization' + ? canonicalizationDigestProvider + : undefined, + ); } catch { throw new Error('Measured revision-evidence execution failed.'); } @@ -352,11 +376,12 @@ async function runMeasuredEvidence(createEvidence, source, operation) { throw new Error('invalid revision evidence'); } revisions = - operation === 'revision' - ? [evidence.revision] - : [evidence.previousRevision, evidence.resultingRevision]; + operation === 'transition' + ? [evidence.previousRevision, evidence.resultingRevision] + : [evidence.revision]; if ( (operation === 'transition' && typeof evidence.changed !== 'boolean') || + (operation === 'canonicalization' && digestCalls !== 1) || revisions.some( (revision) => typeof revision !== 'object' || revision === null, ) || @@ -398,15 +423,15 @@ async function main() { const measuredModule = await loadMeasuredModule(modulePath); const createEvidence = - args.operation === 'revision' - ? measuredModule.createDocumentEnvelopeRevisionEvidenceBytes - : measuredModule.createDocumentEnvelopeTransitionEvidenceBytes; + args.operation === 'transition' + ? measuredModule.createDocumentEnvelopeTransitionEvidenceBytes + : measuredModule.createDocumentEnvelopeRevisionEvidenceBytes; if (typeof createEvidence !== 'function') { throw new Error( `Measured revision module must export ${ - args.operation === 'revision' - ? 'createDocumentEnvelopeRevisionEvidenceBytes' - : 'createDocumentEnvelopeTransitionEvidenceBytes' + args.operation === 'transition' + ? 'createDocumentEnvelopeTransitionEvidenceBytes' + : 'createDocumentEnvelopeRevisionEvidenceBytes' }().`, ); } @@ -442,7 +467,10 @@ async function main() { `${JSON.stringify( { contractVersion: 1, - benchmarkId: `${args.operation}-evidence-${args.profile}`, + benchmarkId: + args.operation === 'canonicalization' + ? `envelope-canonicalization-${args.profile}` + : `${args.operation}-evidence-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index 7803ec4a..644b3823 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -568,6 +568,16 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) 'transition', 'summary', ); + const canonicalizationSamplesPath = resolve( + args.outputDirectory, + 'envelope-canonicalization', + 'samples.json', + ); + const canonicalizationSummaryDirectory = resolve( + args.outputDirectory, + 'envelope-canonicalization', + 'summary', + ); runMeasurementAndSummary({ measurementScript: 'measure-markdown.mjs', @@ -593,8 +603,19 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) measurementFailure: 'Benchmark suite transition measurement failed.', summaryFailure: 'Benchmark suite transition summary failed.', }); - if (autosaveArguments !== null) { + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [ + ...revisionArguments, + '--operation', + 'canonicalization', + ], + samplesPath: canonicalizationSamplesPath, + summaryDirectory: canonicalizationSummaryDirectory, + measurementFailure: 'Benchmark suite envelope canonicalization measurement failed.', + summaryFailure: 'Benchmark suite envelope canonicalization summary failed.', + }); const autosaveSamplesPath = resolve( args.outputDirectory, 'autosave', @@ -672,6 +693,11 @@ function suiteManifest(args, packageEvidence, includeAutosave) { transitionSummaryText: 'transition/summary/summary.txt', ...(includeAutosave ? { + canonicalizationSamples: 'envelope-canonicalization/samples.json', + canonicalizationSummaryJson: + 'envelope-canonicalization/summary/summary.json', + canonicalizationSummaryText: + 'envelope-canonicalization/summary/summary.txt', autosaveSamples: 'autosave/samples.json', autosaveSummaryJson: 'autosave/summary/summary.json', autosaveSummaryText: 'autosave/summary/summary.txt', diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index f851ec07..34693c0b 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -73,7 +73,11 @@ function createPackedBenchmarkFixture( writeFileSync( join(distDirectory, 'cwl-revision-evidence.js'), `const revision = { digestHex: '${'e'.repeat(64)}' }; -export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (!provider) return { revision }; + const digestHex = Buffer.from(await provider.digest('SHA-256', source)).toString('hex'); + return { revision: { digestHex } }; +} export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } `, 'utf8', diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index b35aa1bd..70b10712 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -69,7 +69,11 @@ function createPackedBenchmarkFixture(directory: string): { writeFileSync( join(distDirectory, 'cwl-revision-evidence.js'), `const revision = { digestHex: '${'c'.repeat(64)}' }; -export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (!provider) return { revision }; + const digestHex = Buffer.from(await provider.digest('SHA-256', source)).toString('hex'); + return { revision: { digestHex } }; +} export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } `, 'utf8', @@ -205,6 +209,11 @@ describe('packed artifact benchmark suite contract', () => { autosaveCommitSamples: 'autosave-commit/samples.json', autosaveCommitSummaryJson: 'autosave-commit/summary/summary.json', autosaveCommitSummaryText: 'autosave-commit/summary/summary.txt', + canonicalizationSamples: 'envelope-canonicalization/samples.json', + canonicalizationSummaryJson: + 'envelope-canonicalization/summary/summary.json', + canonicalizationSummaryText: + 'envelope-canonicalization/summary/summary.txt', htmlSerializationSamples: 'html-serialization/samples.json', htmlSerializationSummaryJson: 'html-serialization/summary/summary.json', @@ -240,6 +249,17 @@ describe('packed artifact benchmark suite contract', () => { expect(commitSamples.benchmarkId).toBe('autosave-commit-small'); expect(commitSamples.samples).toHaveLength(2); + const canonicalizationSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'envelope-canonicalization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(canonicalizationSamples.benchmarkId).toBe( + 'envelope-canonicalization-small', + ); + expect(canonicalizationSamples.samples).toHaveLength(2); + const htmlSamples = JSON.parse( readFileSync( join(directory, 'evidence', 'html-serialization', 'samples.json'), diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 324714d8..043500af 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -49,7 +49,7 @@ function argumentsFor( input: string, modulePath: string, output: string, - operation?: 'transition', + operation?: 'transition' | 'canonicalization', ): string[] { return [ measurementScript, @@ -185,6 +185,38 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { } }); + it('isolates strict envelope canonicalization from digest-provider cost', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-canonicalization-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + const digest = await provider.digest('SHA-256', source); + return { revision: { digestHex: Buffer.from(digest).toString('hex') } }; +}\n`, + 'utf8', + ); + + execFileSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'canonicalization'), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ + benchmarkId: 'envelope-canonicalization-large', + artifactSha256: fileSha256(modulePath), + samples: expect.any(Array), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed before output when the measured module lacks the revision API', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); const input = join(root, 'small.json'); From 8402073e67e6744b62ed1c87fcb804fde49c6527 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:54:41 +0900 Subject: [PATCH 239/260] fix(perf): scale HTML benchmark corpus Signed-off-by: Seongho Bae --- .github/workflows/performance-evidence.yml | 6 +----- benchmarks/corpus.lock.json | 8 ++++++++ benchmarks/generate-corpus.mjs | 12 ++++++++++++ src/performanceCorpusContract.test.ts | 9 +++++++++ src/performanceWorkflowContract.test.ts | 8 +------- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index e173f421..caa28a1d 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -62,15 +62,11 @@ jobs: run: | set -euo pipefail corpus_dir="${RUNNER_TEMP}/inkspan-benchmark-corpus" - html_input="${RUNNER_TEMP}/inkspan-html-input.html" package_dir="${RUNNER_TEMP}/inkspan-package" evidence_dir="${RUNNER_TEMP}/inkspan-performance-evidence" rm -rf "$corpus_dir" "$package_dir" "$evidence_dir" - rm -f "$html_input" - node benchmarks/generate-corpus.mjs --output "$corpus_dir" - printf '%s\n' '

Inkspan deterministic performance smoke

' > "$html_input" mkdir -p "$package_dir" pnpm pack --pack-destination "$package_dir" @@ -83,7 +79,7 @@ jobs: node benchmarks/run-current-suite.mjs \ --input "$corpus_dir/small.md" \ - --html-input "$html_input" \ + --html-input "$corpus_dir/small.html" \ --revision-input "$corpus_dir/small.envelope.json" \ --package-tarball "$package" \ --package-sha256 "$package_sha256" \ diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index 5ee1b96c..01078675 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -14,6 +14,8 @@ "sections": 1, "bytes": 2152, "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea", + "htmlBytes": 2225, + "htmlSha256": "f06ba85c8fa00c819dcb992232fbad2205915a3066abc29e275c590f32fb5356", "envelopeBytes": 4236, "envelopeSha256": "7ee1425e1f57808a3a4d11987a0a6bc2aec97d311457f5456e82f1adbc48c810" }, @@ -21,6 +23,8 @@ "sections": 8, "bytes": 15712, "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c", + "htmlBytes": 15807, + "htmlSha256": "09dcb4384632d6b5525707d21c22788e838f7c68ae5e55e020ff0367e342bc5a", "envelopeBytes": 30018, "envelopeSha256": "47fa0cd11ffec5f9c5513a935180d1d7249157c46a0e115d1de938be86fa4459" }, @@ -28,6 +32,8 @@ "sections": 32, "bytes": 62199, "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392", + "htmlBytes": 62365, + "htmlSha256": "3952dd5228a1e0f34bc294019e317c5ae38a26d4b4606cca8344e4dc36221102", "envelopeBytes": 118409, "envelopeSha256": "c93143659a352cffaa1ff817b0efa0ba4b63f8410bbb7d923799b827a36aa655" }, @@ -35,6 +41,8 @@ "sections": 128, "bytes": 248152, "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928", + "htmlBytes": 248607, + "htmlSha256": "1d5d42bd92b7121a4ca59a542edf881d0edb438d7de68c09d3c075703bf3e4cb", "envelopeBytes": 471978, "envelopeSha256": "ebc47c89671e39af349a4cb65ba15ed3138a3eee187245e520abe80d877bcd6c" } diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index f8011aac..14af0d41 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -150,6 +150,14 @@ function buildEnvelope(body) { })}\n`; } +function buildHtml(profile, body) { + const escapedBody = body + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>'); + return `
${escapedBody}
\n`; +} + function sha256(bytes) { return createHash('sha256').update(bytes).digest('hex'); } @@ -170,8 +178,10 @@ const profileManifest = {}; for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { const body = buildProfile(profile, sections); const bytes = Buffer.from(body, 'utf8'); + const htmlBytes = Buffer.from(buildHtml(profile, body), 'utf8'); const envelopeBytes = Buffer.from(buildEnvelope(body), 'utf8'); writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); + writeRegularOutput(resolve(outputDirectory, `${profile}.html`), htmlBytes); writeRegularOutput( resolve(outputDirectory, `${profile}.envelope.json`), envelopeBytes, @@ -180,6 +190,8 @@ for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { sections, bytes: bytes.byteLength, sha256: sha256(bytes), + htmlBytes: htmlBytes.byteLength, + htmlSha256: sha256(htmlBytes), envelopeBytes: envelopeBytes.byteLength, envelopeSha256: sha256(envelopeBytes), }); diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index e059e59b..f9f07bcf 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -16,6 +16,8 @@ interface BenchmarkProfileLock { readonly sections: number; readonly bytes: number; readonly sha256: string; + readonly htmlBytes: number; + readonly htmlSha256: string; readonly envelopeBytes: number; readonly envelopeSha256: string; } @@ -79,6 +81,13 @@ describe('deterministic synthetic performance corpus', () => { const secondBytes = readFileSync(join(second, `${profile}.md`)); expect(firstBytes.equals(secondBytes)).toBe(true); expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + const firstHtml = readFileSync(join(first, `${profile}.html`)); + const secondHtml = readFileSync(join(second, `${profile}.html`)); + expect(firstHtml.equals(secondHtml)).toBe(true); + expect(firstHtml.byteLength).toBe(expected.profiles[profile].htmlBytes); + expect(firstHtml.toString('utf8')).toContain( + `data-inkspan-benchmark-profile="${profile}"`, + ); const firstEnvelope = readFileSync( join(first, `${profile}.envelope.json`), ); diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index 7a37544e..b0673d39 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -33,13 +33,7 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain('pnpm build'); expect(workflow).toContain('pnpm pack --pack-destination'); expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); - expect(workflow).toContain( - 'html_input="${RUNNER_TEMP}/inkspan-html-input.html"', - ); - expect(workflow).toContain( - "printf '%s\\n' '

Inkspan deterministic performance smoke

' > \"$html_input\"", - ); - expect(workflow).toContain('--html-input "$html_input"'); + expect(workflow).toContain('--html-input "$corpus_dir/small.html"'); expect(workflow).toContain( '--revision-input "$corpus_dir/small.envelope.json"', ); From c05ca75ad174c789f5d0b54148628aae2fbd6ccf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:59:23 +0900 Subject: [PATCH 240/260] fix(perf): bind packed evidence to corpus Signed-off-by: Seongho Bae --- benchmarks/run-current-suite.mjs | 95 +++++++++++++++---- ...ormancePackedArtifactSuiteContract.test.ts | 49 ++++++++-- 2 files changed, 116 insertions(+), 28 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 21ca46e5..6fd95483 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -31,6 +31,11 @@ const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); const PACKED_MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const CORPUS_LOCK_PATH = resolve(benchmarkDirectory, 'corpus.lock.json'); +const MAX_CORPUS_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_CORPUS_LOCK_BYTES = 1024 * 1024; +const CORPUS_MISMATCH_ERROR = + 'Benchmark suite inputs must match the committed corpus profile.'; const legacyFlags = Object.freeze([ '--input', '--module', @@ -158,48 +163,40 @@ function claimedLegacySourceCommitSha(argv) { return undefined; } -function readPackedTarballSnapshot(path) { +function readRegularFileSnapshot(path, maximumBytes, invalidMessage, oversizedMessage) { let pathMetadata; try { pathMetadata = lstatSync(path, { throwIfNoEntry: false }); } catch { - throw new Error( - 'Benchmark suite package tarball must be a regular non-symlink file.', - ); + throw new Error(invalidMessage); } if ( pathMetadata === undefined || pathMetadata.isSymbolicLink() || !pathMetadata.isFile() ) { - throw new Error( - 'Benchmark suite package tarball must be a regular non-symlink file.', - ); + throw new Error(invalidMessage); } let descriptor; try { descriptor = openSync(path, READ_ONLY_NOFOLLOW); } catch { - throw new Error( - 'Benchmark suite package tarball must be a regular non-symlink file.', - ); + throw new Error(invalidMessage); } try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error( - 'Benchmark suite package tarball must be a regular non-symlink file.', - ); + throw new Error(invalidMessage); } - if (metadata.size > MAX_PACKAGE_BYTES) { - throw new Error('Benchmark suite package tarball exceeds the supported size.'); + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); } const chunks = []; let totalBytes = 0; - while (totalBytes <= MAX_PACKAGE_BYTES) { - const remainingBudget = MAX_PACKAGE_BYTES + 1 - totalBytes; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; const chunk = Buffer.allocUnsafe( Math.min(READ_CHUNK_BYTES, remainingBudget), ); @@ -212,8 +209,8 @@ function readPackedTarballSnapshot(path) { ); if (bytesRead === 0) break; totalBytes += bytesRead; - if (totalBytes > MAX_PACKAGE_BYTES) { - throw new Error('Benchmark suite package tarball exceeds the supported size.'); + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); } chunks.push(chunk.subarray(0, bytesRead)); } @@ -223,6 +220,65 @@ function readPackedTarballSnapshot(path) { } } +function readPackedTarballSnapshot(path) { + return readRegularFileSnapshot( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function assertPackedCorpusInputs(argv) { + const flags = matchesArguments(argv, packedHtmlFlags) + ? packedHtmlFlags + : matchesArguments(argv, packedFlags) + ? packedFlags + : null; + if (flags === null) return; + + const values = valuesForArguments(argv, flags); + let lock; + try { + lock = JSON.parse( + readRegularFileSnapshot( + CORPUS_LOCK_PATH, + MAX_CORPUS_LOCK_BYTES, + CORPUS_MISMATCH_ERROR, + CORPUS_MISMATCH_ERROR, + ).toString('utf8'), + ); + } catch { + throw new Error(CORPUS_MISMATCH_ERROR); + } + const profile = lock?.profiles?.[values['--profile']]; + const inputs = [ + ['--input', 'bytes', 'sha256'], + ['--revision-input', 'envelopeBytes', 'envelopeSha256'], + ...(flags === packedHtmlFlags + ? [['--html-input', 'htmlBytes', 'htmlSha256']] + : []), + ]; + try { + for (const [flag, byteKey, digestKey] of inputs) { + const bytes = readRegularFileSnapshot( + values[flag], + MAX_CORPUS_INPUT_BYTES, + CORPUS_MISMATCH_ERROR, + CORPUS_MISMATCH_ERROR, + ); + if ( + bytes.byteLength !== profile?.[byteKey] || + createHash('sha256').update(bytes).digest('hex') !== profile?.[digestKey] + ) { + throw new Error(CORPUS_MISMATCH_ERROR); + } + } + } catch { + throw new Error(CORPUS_MISMATCH_ERROR); + } +} + function snapshotPackedArguments(argv) { const flags = matchesArguments(argv, packedHtmlFlags) ? packedHtmlFlags @@ -546,6 +602,7 @@ function runExistingSuite(argv) { function main(argv) { assertCleanSourceCheckout(repositoryRoot); assertFreshOutputDirectory(argv); + assertPackedCorpusInputs(argv); const expectedLegacySourceCommitSha = claimedLegacySourceCommitSha(argv); if (expectedLegacySourceCommitSha !== undefined) { assertCleanSourceCheckout(repositoryRoot, expectedLegacySourceCommitSha); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 70b10712..bd8a3311 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; import { + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -127,16 +128,19 @@ function packedSuiteArguments(options: { sourceCommitSha?: string; tarballPath: string; }): string[] { - const markdownInputPath = join(options.directory, 'input.md'); - const htmlInputPath = join(options.directory, 'input.html'); - const revisionInputPath = join(options.directory, 'document-envelope.json'); - writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); - writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Packed buyer benchmark"}]}]}}\n', - 'utf8', + const corpusDirectory = join(options.directory, 'corpus'); + execFileSync( + process.execPath, + [ + resolve(repositoryRoot, 'benchmarks/generate-corpus.mjs'), + '--output', + corpusDirectory, + ], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); + const markdownInputPath = join(corpusDirectory, 'small.md'); + const htmlInputPath = join(corpusDirectory, 'small.html'); + const revisionInputPath = join(corpusDirectory, 'small.envelope.json'); return [ suitePath, '--input', @@ -298,6 +302,33 @@ describe('packed artifact benchmark suite contract', () => { ); }); + it('rejects a profile label whose packed-suite inputs do not match the committed corpus', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-corpus-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + const args = packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }); + writeFileSync(join(directory, 'corpus', 'small.html'), '

tiny

\n'); + + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite inputs must match the committed corpus profile.\n', + ); + expect(existsSync(join(directory, 'evidence'))).toBe(false); + }); + it('rejects source provenance that does not match the benchmark checkout', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-source-')); temporaryDirectories.push(directory); From 50f0bff10eab0179e325c2007f52bc1c36f1575a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 01:00:31 +0900 Subject: [PATCH 241/260] test(perf): use locked packed corpus Signed-off-by: Seongho Bae --- ...ormancePackedArtifactPathStability.test.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 34693c0b..b1f1c228 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -171,15 +171,19 @@ describe('packed artifact benchmark path stability', () => { original.tarballPath, adversarial.tarballPath, ); - const markdownInputPath = join(directory, 'input.md'); - const revisionInputPath = join(directory, 'document-envelope.json'); - const outputDirectory = join(directory, 'evidence'); - writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Stable packed artifact"}]}]}}\n', - 'utf8', + const corpusDirectory = join(directory, 'corpus'); + execFileSync( + process.execPath, + [ + resolve(repositoryRoot, 'benchmarks/generate-corpus.mjs'), + '--output', + corpusDirectory, + ], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); + const markdownInputPath = join(corpusDirectory, 'small.md'); + const revisionInputPath = join(corpusDirectory, 'small.envelope.json'); + const outputDirectory = join(directory, 'evidence'); const result = spawnSync( process.execPath, From d9780837f64df40811a960f35dc73aea62419f1b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 01:04:59 +0900 Subject: [PATCH 242/260] ci(perf): add full profile evidence matrix Signed-off-by: Seongho Bae --- .github/workflows/performance-evidence.yml | 26 +++++++++++++--------- src/performanceWorkflowContract.test.ts | 20 ++++++++++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index caa28a1d..a054980c 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -10,12 +10,15 @@ on: - 'pnpm-lock.yaml' - 'tsconfig.json' - 'vite.config.ts' + schedule: + - cron: '17 3 * * 1' + workflow_dispatch: permissions: contents: read concurrency: - group: performance-evidence-${{ github.event.pull_request.number }} + group: performance-evidence-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: @@ -23,20 +26,23 @@ env: jobs: packed-artifact-smoke: - name: packed-artifact-smoke + name: packed-artifact-${{ matrix.profile }} runs-on: ubuntu-24.04 timeout-minutes: 20 + strategy: + fail-fast: false + matrix: ${{ fromJSON(github.event_name == 'pull_request' && '{"include":[{"profile":"small","samples":3}]}' || '{"include":[{"profile":"small","samples":25},{"profile":"medium","samples":25},{"profile":"large","samples":25},{"profile":"stress","samples":25}]}') }} steps: - name: Checkout exact pull-request head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Verify exact checkout shell: bash env: - INKSPAN_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + INKSPAN_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | set -euo pipefail actual_head="$(git rev-parse HEAD)" @@ -78,13 +84,13 @@ jobs: runtime_id="node-$(node -p 'process.versions.node')" node benchmarks/run-current-suite.mjs \ - --input "$corpus_dir/small.md" \ - --html-input "$corpus_dir/small.html" \ - --revision-input "$corpus_dir/small.envelope.json" \ + --input "$corpus_dir/${{ matrix.profile }}.md" \ + --html-input "$corpus_dir/${{ matrix.profile }}.html" \ + --revision-input "$corpus_dir/${{ matrix.profile }}.envelope.json" \ --package-tarball "$package" \ --package-sha256 "$package_sha256" \ - --profile small \ - --samples 3 \ + --profile "${{ matrix.profile }}" \ + --samples "${{ matrix.samples }}" \ --source-commit-sha "$source_sha" \ --runtime-id "$runtime_id" \ --reference-hardware-id github-actions-ubuntu-24.04-x64 \ @@ -93,7 +99,7 @@ jobs: - name: Upload bounded performance evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0.0 with: - name: performance-smoke-${{ github.event.pull_request.head.sha }} + name: performance-${{ matrix.profile }}-${{ github.event.pull_request.head.sha || github.sha }} path: ${{ runner.temp }}/inkspan-performance-evidence if-no-files-found: error retention-days: 5 diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index b0673d39..e219acb7 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -18,6 +18,8 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain('name: Performance Evidence'); expect(workflow).toContain('pull_request:'); + expect(workflow).toContain("cron: '17 3 * * 1'"); + expect(workflow).toContain('workflow_dispatch:'); expect(workflow).toContain('permissions:\n contents: read'); expect(workflow).toContain('runs-on: ubuntu-24.04'); expect(workflow).toContain('timeout-minutes: 20'); @@ -25,7 +27,7 @@ describe('performance evidence workflow contract', () => { 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', ); expect(workflow).toContain( - 'ref: ${{ github.event.pull_request.head.sha }}', + 'ref: ${{ github.event.pull_request.head.sha || github.sha }}', ); expect(workflow).toContain('persist-credentials: false'); expect(workflow).toContain('Verify exact checkout'); @@ -33,13 +35,21 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain('pnpm build'); expect(workflow).toContain('pnpm pack --pack-destination'); expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); - expect(workflow).toContain('--html-input "$corpus_dir/small.html"'); expect(workflow).toContain( - '--revision-input "$corpus_dir/small.envelope.json"', + '"profile":"small","samples":3', + ); + for (const profile of ['small', 'medium', 'large', 'stress']) { + expect(workflow).toContain(`"profile":"${profile}","samples":25`); + } + expect(workflow).toContain( + '--html-input "$corpus_dir/${{ matrix.profile }}.html"', + ); + expect(workflow).toContain( + '--revision-input "$corpus_dir/${{ matrix.profile }}.envelope.json"', ); expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); - expect(workflow).toContain('--profile small'); - expect(workflow).toContain('--samples 3'); + expect(workflow).toContain('--profile "${{ matrix.profile }}"'); + expect(workflow).toContain('--samples "${{ matrix.samples }}"'); expect(workflow).toContain( '--reference-hardware-id github-actions-ubuntu-24.04-x64', ); From 699808ee57e4a859fcbdd46f4d95230fac880e79 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 04:42:50 +0900 Subject: [PATCH 243/260] test(perf): require repository-scoped concurrency --- src/performanceWorkflowContract.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index e219acb7..92fa1c34 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -21,6 +21,10 @@ describe('performance evidence workflow contract', () => { expect(workflow).toContain("cron: '17 3 * * 1'"); expect(workflow).toContain('workflow_dispatch:'); expect(workflow).toContain('permissions:\n contents: read'); + expect(workflow).toContain( + "group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }}", + ); + expect(workflow).toContain('cancel-in-progress: true'); expect(workflow).toContain('runs-on: ubuntu-24.04'); expect(workflow).toContain('timeout-minutes: 20'); expect(workflow).toContain( From 5701b7edfdbf76fc3537d81533d4e93944392b62 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 04:43:03 +0900 Subject: [PATCH 244/260] fix(perf): scope concurrency to repository --- .github/workflows/performance-evidence.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index a054980c..a9302f46 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -18,7 +18,7 @@ permissions: contents: read concurrency: - group: performance-evidence-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: From 9258bdbdf8ec8ca87ab25b576fe2599c88ba9040 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 07:28:50 +0900 Subject: [PATCH 245/260] experiment: reuse canonical UTF-8 encoder Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Claude (via Claude Code) --- src/documentEnvelopeCanonical.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/documentEnvelopeCanonical.ts b/src/documentEnvelopeCanonical.ts index c4dffea6..3fdc04da 100644 --- a/src/documentEnvelopeCanonical.ts +++ b/src/documentEnvelopeCanonical.ts @@ -20,6 +20,7 @@ const INVALID_UNICODE_MESSAGE = 'Document envelope must contain valid Unicode scalar strings'; const NEGATIVE_ZERO_MESSAGE = 'Document envelope must not contain negative zero'; +const UTF8_ENCODER = new TextEncoder(); /** * Serialize a valid Inkspan envelope to deterministic RFC 8785 JSON. @@ -58,7 +59,7 @@ export function serializeValidatedDocumentEnvelope( export function encodeValidatedDocumentEnvelope( envelope: CwlEditorDocumentEnvelope, ): Uint8Array { - return new TextEncoder().encode( + return UTF8_ENCODER.encode( serializeValidatedDocumentEnvelope(envelope), ); } From e5ff9c3802d7105904b8326f659bf2c86983f238 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 16:16:27 +0900 Subject: [PATCH 246/260] test(perf): validate changed document transition scenarios Preserve the no-op baseline and add an explicit before/after corpus, packed suite lane, and scenario correctness gates. Signed-off-by: Seongho Bae --- .github/workflows/performance-evidence.yml | 1 + benchmarks/README.md | 39 ++++++ benchmarks/compare-summaries.mjs | 2 +- benchmarks/corpus.lock.json | 16 ++- benchmarks/generate-corpus.mjs | 9 ++ benchmarks/measure-revision-evidence.mjs | 47 +++++-- benchmarks/run-current-suite-core.mjs | 29 ++++- benchmarks/run-current-suite.mjs | 39 +++--- benchmarks/summarize-samples.mjs | 2 +- src/performanceCorpusContract.test.ts | 11 ++ ...ormancePackedArtifactSuiteContract.test.ts | 55 ++++++-- ...rmanceRegressionComparatorContract.test.ts | 23 +++- ...ormanceRevisionMeasurementContract.test.ts | 122 +++++++++++++++++- src/performanceWorkflowContract.test.ts | 3 + 14 files changed, 347 insertions(+), 51 deletions(-) create mode 100644 benchmarks/README.md diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml index a9302f46..a2bc00a8 100644 --- a/.github/workflows/performance-evidence.yml +++ b/.github/workflows/performance-evidence.yml @@ -94,6 +94,7 @@ jobs: --source-commit-sha "$source_sha" \ --runtime-id "$runtime_id" \ --reference-hardware-id github-actions-ubuntu-24.04-x64 \ + --resulting-input "$corpus_dir/${{ matrix.profile }}.changed.envelope.json" \ --output "$evidence_dir" - name: Upload bounded performance evidence diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..8e87bff0 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,39 @@ +# Performance evidence development harness + +This harness is research infrastructure, not a published latency or supported +document-size guarantee. The generated corpus is synthetic and suitable for +repeatable regression probes and harness contract tests. It does not establish +buyer-workload performance, real-device input behavior, or the 20 ms target. +Envelope fixtures contain plain paragraphs; Markdown list, table, and image +syntax inside those paragraphs is not a rich editor document tree. + +## Transition scenarios + +| Operation | Inputs | Required result | Metric prefix | +| --- | --- | --- | --- | +| `transition` | Same captured envelope twice | Unchanged, equal revision digests | `transition-evidence` | +| `transition-changed` | Explicit previous and resulting envelopes | Changed, unequal revision digests | `transition-changed-evidence` | + +The profile suffix remains `small`, `medium`, `large`, or `stress`. Existing +transition samples retain their no-op meaning. Never compare the two scenarios +as an optimization result; the comparison tool rejects mismatched metric IDs. + +For `measure-revision-evidence.mjs`, append +`--operation transition-changed --resulting-input ` +immediately before `--output`. Other operations reject `--resulting-input`. +Both inputs use the same bounded, non-symlink file reader. An output cannot +overwrite either input, including through a hard link. Invalid results publish +no sample file, and sample output contains neither document content nor paths. + +The packed `run-current-suite.mjs` accepts `--resulting-input` immediately before +`--output`, with or without `--html-input`. It verifies the resulting fixture +against `changedEnvelopeBytes` and `changedEnvelopeSha256` in `corpus.lock.json`. +The generator adds one plain paragraph to produce each +`.changed.envelope.json`; all earlier fixture bytes and digests remain +unchanged. Omitting the new flag preserves the earlier suite. The +`performance-evidence.yml` workflow exercises both transition scenarios. + +Commit a clean source checkout and rebuild the package before recording evidence. +Keep raw samples with the exact source SHA, packed-artifact digest, Node runtime, +hardware identity, and sample count. A new scenario starts a separate baseline; +synthetic smoke evidence cannot close the realistic-workload support-envelope gap. diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index a5c252da..b75f9d0b 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -16,7 +16,7 @@ const READ_ONLY_NONBLOCKING = (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = - /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|transition-changed-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index 01078675..2ec36807 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -17,7 +17,9 @@ "htmlBytes": 2225, "htmlSha256": "f06ba85c8fa00c819dcb992232fbad2205915a3066abc29e275c590f32fb5356", "envelopeBytes": 4236, - "envelopeSha256": "7ee1425e1f57808a3a4d11987a0a6bc2aec97d311457f5456e82f1adbc48c810" + "envelopeSha256": "7ee1425e1f57808a3a4d11987a0a6bc2aec97d311457f5456e82f1adbc48c810", + "changedEnvelopeBytes": 4319, + "changedEnvelopeSha256": "aafa73ad4ed397de59a55fd21cd8c021dfee9eb2a828e732dc693d040c13ba64" }, "medium": { "sections": 8, @@ -26,7 +28,9 @@ "htmlBytes": 15807, "htmlSha256": "09dcb4384632d6b5525707d21c22788e838f7c68ae5e55e020ff0367e342bc5a", "envelopeBytes": 30018, - "envelopeSha256": "47fa0cd11ffec5f9c5513a935180d1d7249157c46a0e115d1de938be86fa4459" + "envelopeSha256": "47fa0cd11ffec5f9c5513a935180d1d7249157c46a0e115d1de938be86fa4459", + "changedEnvelopeBytes": 30101, + "changedEnvelopeSha256": "68cc2e64e3129cfdd49b9fd16f2bea134f6f7275a8e671873f5913d4adde12ab" }, "large": { "sections": 32, @@ -35,7 +39,9 @@ "htmlBytes": 62365, "htmlSha256": "3952dd5228a1e0f34bc294019e317c5ae38a26d4b4606cca8344e4dc36221102", "envelopeBytes": 118409, - "envelopeSha256": "c93143659a352cffaa1ff817b0efa0ba4b63f8410bbb7d923799b827a36aa655" + "envelopeSha256": "c93143659a352cffaa1ff817b0efa0ba4b63f8410bbb7d923799b827a36aa655", + "changedEnvelopeBytes": 118492, + "changedEnvelopeSha256": "d95afab352b1813395f7e95bd8c2c8c983583ca1577e1ae56fee60e961516bbf" }, "stress": { "sections": 128, @@ -44,7 +50,9 @@ "htmlBytes": 248607, "htmlSha256": "1d5d42bd92b7121a4ca59a542edf881d0edb438d7de68c09d3c075703bf3e4cb", "envelopeBytes": 471978, - "envelopeSha256": "ebc47c89671e39af349a4cb65ba15ed3138a3eee187245e520abe80d877bcd6c" + "envelopeSha256": "ebc47c89671e39af349a4cb65ba15ed3138a3eee187245e520abe80d877bcd6c", + "changedEnvelopeBytes": 472061, + "changedEnvelopeSha256": "4a160cde5bf9c7ac9aa8aeeae0bce13d10632bd055944dc7c61a70403c55746a" } } } diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index 14af0d41..74971e6e 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -180,12 +180,19 @@ for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { const bytes = Buffer.from(body, 'utf8'); const htmlBytes = Buffer.from(buildHtml(profile, body), 'utf8'); const envelopeBytes = Buffer.from(buildEnvelope(body), 'utf8'); + const changedEnvelopeBytes = Buffer.from( + buildEnvelope(`${body}\nSynthetic appended edit.`), 'utf8', + ); writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); writeRegularOutput(resolve(outputDirectory, `${profile}.html`), htmlBytes); writeRegularOutput( resolve(outputDirectory, `${profile}.envelope.json`), envelopeBytes, ); + writeRegularOutput( + resolve(outputDirectory, `${profile}.changed.envelope.json`), + changedEnvelopeBytes, + ); profileManifest[profile] = Object.freeze({ sections, bytes: bytes.byteLength, @@ -194,6 +201,8 @@ for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { htmlSha256: sha256(htmlBytes), envelopeBytes: envelopeBytes.byteLength, envelopeSha256: sha256(envelopeBytes), + changedEnvelopeBytes: changedEnvelopeBytes.byteLength, + changedEnvelopeSha256: sha256(changedEnvelopeBytes), }); } diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 4eb32ba4..8c20e302 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -48,8 +48,13 @@ function resolveArguments(argv) { '--output', ]; const operationFlags = [...baseFlags.slice(0, -1), '--operation', '--output']; + const changedTransitionFlags = [ + ...operationFlags.slice(0, -1), '--resulting-input', '--output', + ]; const expectedFlags = - argv.length === operationFlags.length * 2 ? operationFlags : baseFlags; + [baseFlags, operationFlags, changedTransitionFlags].find( + (flags) => argv.length === flags.length * 2, + ) ?? baseFlags; if ( argv.length !== expectedFlags.length * 2 || expectedFlags.some((flag, index) => argv[index * 2] !== flag) || @@ -99,13 +104,19 @@ function resolveArguments(argv) { if ( operation !== 'revision' && operation !== 'transition' && + operation !== 'transition-changed' && operation !== 'canonicalization' ) { throw new Error('Revision benchmark operation is invalid.'); } + if ((operation === 'transition-changed') !== (values['--resulting-input'] !== undefined)) { + throw new Error('Only changed-transition measurement requires a resulting input.'); + } return Object.freeze({ inputPath: resolve(values['--input']), + resultingInputPath: values['--resulting-input'] === undefined + ? undefined : resolve(values['--resulting-input']), modulePath: values['--module'], profile, sampleCount, @@ -339,7 +350,8 @@ function writeMeasurementOutput(path, content) { } } -async function runMeasuredEvidence(createEvidence, source, operation) { +async function runMeasuredEvidence(createEvidence, source, resultingSource, operation) { + const isTransition = operation === 'transition' || operation === 'transition-changed'; let digestCalls = 0; const canonicalizationDigestProvider = { digest(algorithm, canonicalSource) { @@ -357,8 +369,8 @@ async function runMeasuredEvidence(createEvidence, source, operation) { let evidence; try { evidence = - operation === 'transition' - ? await createEvidence(source, source) + isTransition + ? await createEvidence(source, resultingSource) : await createEvidence( source, undefined, @@ -376,11 +388,11 @@ async function runMeasuredEvidence(createEvidence, source, operation) { throw new Error('invalid revision evidence'); } revisions = - operation === 'transition' + isTransition ? [evidence.previousRevision, evidence.resultingRevision] : [evidence.revision]; if ( - (operation === 'transition' && typeof evidence.changed !== 'boolean') || + (isTransition && evidence.changed !== (operation === 'transition-changed')) || (operation === 'canonicalization' && digestCalls !== 1) || revisions.some( (revision) => typeof revision !== 'object' || revision === null, @@ -393,6 +405,9 @@ async function runMeasuredEvidence(createEvidence, source, operation) { ) { throw new Error('invalid revision evidence'); } + if (isTransition && (revisions[0].digestHex !== revisions[1].digestHex) !== evidence.changed) { + throw new Error('invalid transition revision pair'); + } } catch { throw new Error('Measured revision-evidence result is invalid.'); } @@ -403,12 +418,21 @@ async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); const source = readBoundedEnvelopeBytes(args.inputPath); + const resultingSource = args.resultingInputPath === undefined + ? source : readBoundedEnvelopeBytes(args.resultingInputPath); if ( args.inputPath === args.outputPath || - refersToSameFile(args.inputPath, args.outputPath) + refersToSameFile(args.inputPath, args.outputPath) || + (args.resultingInputPath !== undefined && ( + args.resultingInputPath === args.outputPath || + refersToSameFile(args.resultingInputPath, args.outputPath) + )) ) { throw new Error('Revision benchmark output must not overwrite its input.'); } + if (args.operation === 'transition-changed' && source.equals(resultingSource)) { + throw new Error('Changed-transition benchmark inputs must differ.'); + } const modulePath = resolveLocalModule(args.modulePath); if ( modulePath === args.outputPath || @@ -422,26 +446,27 @@ async function main() { assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); + const isTransition = args.operation === 'transition' || args.operation === 'transition-changed'; const createEvidence = - args.operation === 'transition' + isTransition ? measuredModule.createDocumentEnvelopeTransitionEvidenceBytes : measuredModule.createDocumentEnvelopeRevisionEvidenceBytes; if (typeof createEvidence !== 'function') { throw new Error( `Measured revision module must export ${ - args.operation === 'transition' + isTransition ? 'createDocumentEnvelopeTransitionEvidenceBytes' : 'createDocumentEnvelopeRevisionEvidenceBytes' }().`, ); } - await runMeasuredEvidence(createEvidence, source, args.operation); + await runMeasuredEvidence(createEvidence, source, resultingSource, args.operation); const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - await runMeasuredEvidence(createEvidence, source, args.operation); + await runMeasuredEvidence(createEvidence, source, resultingSource, args.operation); const elapsed = performance.now() - start; if (!Number.isFinite(elapsed) || elapsed < 0) { throw new Error('Revision measurement produced invalid runtime evidence.'); diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index 644b3823..6d0474ef 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -44,6 +44,9 @@ const packedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const packedChangedFlags = Object.freeze([ + ...packedFlags.slice(0, -1), '--resulting-input', '--output', +]); const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; @@ -85,6 +88,7 @@ function sharedArguments(values) { referenceHardwareId: values['--reference-hardware-id'], markdownInputPath: values['--input'], revisionInputPath: values['--revision-input'], + resultingInputPath: values['--resulting-input'], outputDirectory: resolve(values['--output']), }); } @@ -169,8 +173,11 @@ function currentCheckoutSha() { } function resolveArguments(argv) { - if (matchesArguments(argv, packedFlags)) { - const values = valuesForArguments(argv, packedFlags); + const matchedPackedFlags = [packedFlags, packedChangedFlags].find( + (flags) => matchesArguments(argv, flags), + ); + if (matchedPackedFlags !== undefined) { + const values = valuesForArguments(argv, matchedPackedFlags); const packageSha256 = values['--package-sha256']; if (!SHA256_PATTERN.test(packageSha256)) { throw new Error( @@ -604,6 +611,19 @@ function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) summaryFailure: 'Benchmark suite transition summary failed.', }); if (autosaveArguments !== null) { + if (args.resultingInputPath !== undefined) { + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [ + ...revisionArguments, '--operation', 'transition-changed', + '--resulting-input', args.resultingInputPath, + ], + samplesPath: resolve(args.outputDirectory, 'transition-changed', 'samples.json'), + summaryDirectory: resolve(args.outputDirectory, 'transition-changed', 'summary'), + measurementFailure: 'Benchmark suite changed-transition measurement failed.', + summaryFailure: 'Benchmark suite changed-transition summary failed.', + }); + } runMeasurementAndSummary({ measurementScript: 'measure-revision-evidence.mjs', measurementArguments: [ @@ -691,6 +711,11 @@ function suiteManifest(args, packageEvidence, includeAutosave) { transitionSamples: 'transition/samples.json', transitionSummaryJson: 'transition/summary/summary.json', transitionSummaryText: 'transition/summary/summary.txt', + ...(args.resultingInputPath === undefined ? {} : { + changedTransitionSamples: 'transition-changed/samples.json', + changedTransitionSummaryJson: 'transition-changed/summary/summary.json', + changedTransitionSummaryText: 'transition-changed/summary/summary.txt', + }), ...(includeAutosave ? { canonicalizationSamples: 'envelope-canonicalization/samples.json', diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 6fd95483..f0869af9 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -90,6 +90,12 @@ const packedHtmlFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const packedChangedFlags = Object.freeze([ + ...packedFlags.slice(0, -1), '--resulting-input', '--output', +]); +const packedHtmlChangedFlags = Object.freeze([ + ...packedHtmlFlags.slice(0, -1), '--resulting-input', '--output', +]); function matchesArguments(argv, expectedFlags) { return ( @@ -110,6 +116,8 @@ function argumentsForFlags(values, flags) { } function matchingFlags(argv) { + if (matchesArguments(argv, packedChangedFlags)) return packedChangedFlags; + if (matchesArguments(argv, packedHtmlChangedFlags)) return packedHtmlChangedFlags; if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; if (matchesArguments(argv, packedHtmlFlags)) return packedHtmlFlags; if (matchesArguments(argv, packedFlags)) return packedFlags; @@ -230,12 +238,8 @@ function readPackedTarballSnapshot(path) { } function assertPackedCorpusInputs(argv) { - const flags = matchesArguments(argv, packedHtmlFlags) - ? packedHtmlFlags - : matchesArguments(argv, packedFlags) - ? packedFlags - : null; - if (flags === null) return; + const flags = matchingFlags(argv); + if (!flags?.includes('--package-tarball')) return; const values = valuesForArguments(argv, flags); let lock; @@ -255,9 +259,12 @@ function assertPackedCorpusInputs(argv) { const inputs = [ ['--input', 'bytes', 'sha256'], ['--revision-input', 'envelopeBytes', 'envelopeSha256'], - ...(flags === packedHtmlFlags + ...(flags.includes('--html-input') ? [['--html-input', 'htmlBytes', 'htmlSha256']] : []), + ...(flags.includes('--resulting-input') + ? [['--resulting-input', 'changedEnvelopeBytes', 'changedEnvelopeSha256']] + : []), ]; try { for (const [flag, byteKey, digestKey] of inputs) { @@ -280,12 +287,8 @@ function assertPackedCorpusInputs(argv) { } function snapshotPackedArguments(argv) { - const flags = matchesArguments(argv, packedHtmlFlags) - ? packedHtmlFlags - : matchesArguments(argv, packedFlags) - ? packedFlags - : null; - if (flags === null) { + const flags = matchingFlags(argv); + if (!flags?.includes('--package-tarball')) { return Object.freeze({ argv, temporaryDirectory: null }); } @@ -486,9 +489,11 @@ function runHtmlSerializationSuite(argv) { } function runPackedHtmlSerializationSuite(argv) { - const values = valuesForArguments(argv, packedHtmlFlags); + const flags = matchingFlags(argv); + const values = valuesForArguments(argv, flags); const outputDirectory = resolve(repositoryRoot, values['--output']); - const coreArguments = argumentsForFlags(values, packedFlags); + const coreFlags = flags.includes('--resulting-input') ? packedChangedFlags : packedFlags; + const coreArguments = argumentsForFlags(values, coreFlags); const snapshotted = snapshotPackedArguments(coreArguments); let coreCompleted = false; @@ -496,7 +501,7 @@ function runPackedHtmlSerializationSuite(argv) { const coreStdout = runCoreNodePreservingError(snapshotted.argv); coreCompleted = true; const manifest = parseCoreManifest(coreStdout); - const snapshotValues = valuesForArguments(snapshotted.argv, packedFlags); + const snapshotValues = valuesForArguments(snapshotted.argv, coreFlags); const snapshotTarballPath = snapshotValues['--package-tarball']; const markdownModuleBytes = readPackedMarkdownModule(snapshotTarballPath); const markdownModulePath = join( @@ -611,7 +616,7 @@ function main(argv) { runHtmlSerializationSuite(argv); return; } - if (matchesArguments(argv, packedHtmlFlags)) { + if (matchesArguments(argv, packedHtmlFlags) || matchesArguments(argv, packedHtmlChangedFlags)) { runPackedHtmlSerializationSuite(argv); return; } diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index b98d2b87..80eb03c9 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -20,7 +20,7 @@ const READ_ONLY_NONBLOCKING_NOFOLLOW = (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = - /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|transition-changed-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index f9f07bcf..cc24431b 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -20,6 +20,8 @@ interface BenchmarkProfileLock { readonly htmlSha256: string; readonly envelopeBytes: number; readonly envelopeSha256: string; + readonly changedEnvelopeBytes: number; + readonly changedEnvelopeSha256: string; } interface BenchmarkCorpusLock { @@ -98,6 +100,15 @@ describe('deterministic synthetic performance corpus', () => { expect(firstEnvelope.byteLength).toBe( expected.profiles[profile].envelopeBytes, ); + const changedEnvelope = readFileSync(join(first, `${profile}.changed.envelope.json`)); + expect(changedEnvelope.equals(readFileSync(join(second, `${profile}.changed.envelope.json`)))).toBe(true); + expect(changedEnvelope.byteLength).toBe(expected.profiles[profile].changedEnvelopeBytes); + expect(expected.profiles[profile].changedEnvelopeSha256).not.toBe(expected.profiles[profile].envelopeSha256); + const edited = JSON.parse(changedEnvelope.toString('utf8')); + expect(edited.documentJson.content.pop()).toEqual({ + type: 'paragraph', content: [{ type: 'text', text: 'Synthetic appended edit.' }], + }); + expect(edited).toEqual(JSON.parse(firstEnvelope.toString('utf8'))); expect( JSON.parse(firstEnvelope.toString('utf8')).documentJson.content.map( (node: { content?: readonly [{ text: string }] }) => diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index bd8a3311..e891b0bf 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -69,13 +69,18 @@ function createPackedBenchmarkFixture(directory: string): { ); writeFileSync( join(distDirectory, 'cwl-revision-evidence.js'), - `const revision = { digestHex: '${'c'.repeat(64)}' }; + `import { createHash } from 'node:crypto'; +const revision = { digestHex: '${'c'.repeat(64)}' }; export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { if (!provider) return { revision }; const digestHex = Buffer.from(await provider.digest('SHA-256', source)).toString('hex'); return { revision: { digestHex } }; } -export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, resulting) { + const previousRevision = { digestHex: createHash('sha256').update(previous).digest('hex') }; + const resultingRevision = { digestHex: createHash('sha256').update(resulting).digest('hex') }; + return { previousRevision, resultingRevision, changed: previousRevision.digestHex !== resultingRevision.digestHex }; +} `, 'utf8', ); @@ -127,6 +132,8 @@ function packedSuiteArguments(options: { runtimeId: string; sourceCommitSha?: string; tarballPath: string; + includeChangedTransition?: boolean; + includeHtml?: boolean; }): string[] { const corpusDirectory = join(options.directory, 'corpus'); execFileSync( @@ -145,8 +152,7 @@ function packedSuiteArguments(options: { suitePath, '--input', markdownInputPath, - '--html-input', - htmlInputPath, + ...(options.includeHtml === false ? [] : ['--html-input', htmlInputPath]), '--revision-input', revisionInputPath, '--package-tarball', @@ -163,13 +169,22 @@ function packedSuiteArguments(options: { options.runtimeId, '--reference-hardware-id', referenceHardwareId, + ...(options.includeChangedTransition ? [ + '--resulting-input', join(corpusDirectory, 'small.changed.envelope.json'), + ] : []), '--output', join(options.directory, 'evidence'), ]; } describe('packed artifact benchmark suite contract', () => { - it('binds one-command benchmark evidence to packed artifact and run provenance', () => { + it.each([ + { includeChangedTransition: false, includeHtml: true }, + { includeChangedTransition: true, includeHtml: true }, + { includeChangedTransition: true, includeHtml: false }, + ])('binds packed evidence to run provenance ($includeChangedTransition, $includeHtml)', ({ + includeChangedTransition, includeHtml, + }) => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); temporaryDirectories.push(directory); const packed = createPackedBenchmarkFixture(directory); @@ -178,6 +193,8 @@ describe('packed artifact benchmark suite contract', () => { process.execPath, packedSuiteArguments({ directory, + includeChangedTransition, + includeHtml, packageSha256: packed.packageSha256, runtimeId: activeRuntimeId, tarballPath: packed.tarballPath, @@ -218,11 +235,11 @@ describe('packed artifact benchmark suite contract', () => { 'envelope-canonicalization/summary/summary.json', canonicalizationSummaryText: 'envelope-canonicalization/summary/summary.txt', - htmlSerializationSamples: 'html-serialization/samples.json', + ...(includeHtml ? { htmlSerializationSamples: 'html-serialization/samples.json', htmlSerializationSummaryJson: 'html-serialization/summary/summary.json', htmlSerializationSummaryText: - 'html-serialization/summary/summary.txt', + 'html-serialization/summary/summary.txt' } : {}), status: 'completed', }); @@ -264,7 +281,22 @@ describe('packed artifact benchmark suite contract', () => { ); expect(canonicalizationSamples.samples).toHaveLength(2); - const htmlSamples = JSON.parse( + if (includeChangedTransition) { + expect(JSON.parse(result.stdout)).toMatchObject({ + changedTransitionSamples: 'transition-changed/samples.json', + changedTransitionSummaryJson: 'transition-changed/summary/summary.json', + changedTransitionSummaryText: 'transition-changed/summary/summary.txt', + }); + const changedSamples = JSON.parse(readFileSync( + join(directory, 'evidence', 'transition-changed', 'samples.json'), 'utf8', + )); + expect(changedSamples.benchmarkId).toBe('transition-changed-evidence-small'); + expect(changedSamples.samples).toHaveLength(2); + } else { + expect(JSON.parse(result.stdout)).not.toHaveProperty('changedTransitionSamples'); + } + if (includeHtml) { + const htmlSamples = JSON.parse( readFileSync( join(directory, 'evidence', 'html-serialization', 'samples.json'), 'utf8', @@ -272,6 +304,7 @@ describe('packed artifact benchmark suite contract', () => { ) as { benchmarkId?: unknown; samples?: unknown[] }; expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); expect(htmlSamples.samples).toHaveLength(2); + } }, 20_000); it('rejects a runtime identifier that does not match the active Node process', () => { @@ -302,17 +335,19 @@ describe('packed artifact benchmark suite contract', () => { ); }); - it('rejects a profile label whose packed-suite inputs do not match the committed corpus', () => { + it.each(['small.html', 'small.changed.envelope.json'])( + 'rejects %s when it does not match the committed corpus', (fileName) => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-corpus-')); temporaryDirectories.push(directory); const packed = createPackedBenchmarkFixture(directory); const args = packedSuiteArguments({ directory, + includeChangedTransition: true, packageSha256: packed.packageSha256, runtimeId: activeRuntimeId, tarballPath: packed.tarballPath, }); - writeFileSync(join(directory, 'corpus', 'small.html'), '

tiny

\n'); + writeFileSync(join(directory, 'corpus', fileName), '

tiny

\n'); const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index d51ae9f1..e98860fc 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -75,13 +75,14 @@ function runComparison( } describe('benchmark regression comparator contract', () => { - it('passes only when a current exact-context metric stays within an explicit tolerance', () => { + it.each(['editor-input-large', 'transition-changed-evidence-large'])( + 'compares only exact-context %s evidence with an explicit tolerance', (benchmarkId) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); try { const result = runComparison( root, - summary(), - summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + summary({ benchmarkId }), + summary({ benchmarkId, artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), '5', ); @@ -89,7 +90,7 @@ describe('benchmark regression comparator contract', () => { expect(result.stderr).toBe(''); expect(JSON.parse(result.stdout)).toEqual({ contractVersion: 1, - benchmarkId: 'editor-input-large', + benchmarkId, unit: 'ms', documentProfile: 'large', runtimeId: 'chromium-1.62.0', @@ -112,6 +113,20 @@ describe('benchmark regression comparator contract', () => { } }); + it('rejects comparisons between changed and unchanged transition scenarios', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-compare-')); + try { + const result = runComparison(root, + summary({ benchmarkId: 'transition-evidence-large' }), + summary({ benchmarkId: 'transition-changed-evidence-large', artifactSha256: CURRENT_ARTIFACT_SHA256 }), '5'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summaries are not comparable: benchmarkId differs.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails a material unapproved regression without hiding the measured receipt', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); try { diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 043500af..17b09299 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -2,9 +2,11 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { existsSync, + linkSync, mkdtempSync, readFileSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -49,7 +51,8 @@ function argumentsFor( input: string, modulePath: string, output: string, - operation?: 'transition' | 'canonicalization', + operation?: 'transition' | 'transition-changed' | 'canonicalization', + resultingInput?: string, ): string[] { return [ measurementScript, @@ -70,6 +73,7 @@ function argumentsFor( '--reference-hardware-id', HARDWARE_ID, ...(operation === undefined ? [] : ['--operation', operation]), + ...(resultingInput === undefined ? [] : ['--resulting-input', resultingInput]), '--output', output, ]; @@ -185,6 +189,122 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { } }); + it.each([ + { operation: 'transition-changed', changed: true, equalDigests: false, valid: true }, + { operation: 'transition-changed', changed: false, equalDigests: true, valid: false }, + { operation: 'transition-changed', changed: true, equalDigests: true, valid: false }, + { operation: 'transition-changed', changed: false, equalDigests: false, valid: false }, + { operation: 'transition', changed: true, equalDigests: false, valid: false }, + { operation: 'transition', changed: false, equalDigests: false, valid: false }, + ] as const)('checks the $operation scenario oracle ($changed, $equalDigests)', ({ + operation, changed, equalDigests, valid, + }) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-changed-transition-')); + const input = join(root, 'previous.json'); + const resultingInput = join(root, 'resulting.json'); + const modulePath = join(root, 'revision.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(resultingInput, readFileSync(input, 'utf8').replace( + 'Synthetic benchmark content', 'Synthetic benchmark content with an edit', + )); + writeFileSync(modulePath, ` +export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, resulting) { + if (previous.toString() !== ${JSON.stringify(readFileSync(input, 'utf8'))}) throw new Error('wrong previous source'); + if (resulting.toString() !== ${JSON.stringify(readFileSync(operation === 'transition' ? input : resultingInput, 'utf8'))}) throw new Error('wrong resulting source'); + return { previousRevision: { digestHex: '${'a'.repeat(64)}' }, resultingRevision: { digestHex: '${(equalDigests ? 'a' : 'b').repeat(64)}' }, changed: ${changed} }; +} +`); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, samplesPath, operation, + operation === 'transition-changed' ? resultingInput : undefined, + ), { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(valid ? 0 : 1); + expect(result.stdout).toBe(''); + expect(existsSync(samplesPath)).toBe(valid); + if (valid) { + const output = readFileSync(samplesPath, 'utf8'); + expect(JSON.parse(output)).toMatchObject({ + benchmarkId: 'transition-changed-evidence-large', + samples: expect.any(Array), + }); + expect(JSON.parse(output).samples).toHaveLength(3); + expect(output).not.toContain('Synthetic benchmark content'); + expect(output).not.toContain(root); + execFileSync(process.execPath, [summaryScript, '--input', samplesPath, + '--output', join(root, 'summary')], { stdio: 'pipe' }); + } else { + expect(result.stderr.trim()).toBe('Measured revision-evidence result is invalid.'); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each(['missing', 'symlink', 'identical', 'overwrite', 'hardlink'] as const)( + 'rejects a %s resulting input without publishing samples or changing input', + (scenario) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-resulting-input-')); + const input = join(root, 'previous.json'); + const resultingInput = join(root, 'private-resulting.json'); + const modulePath = join(root, 'revision.mjs'); + let samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'throw new Error("module must not execute");'); + if (scenario === 'symlink') symlinkSync(input, resultingInput); + else if (scenario !== 'missing') { + writeFileSync(resultingInput, scenario === 'identical' + ? readFileSync(input) : 'private resulting source'); + } + if (scenario === 'overwrite') samplesPath = resultingInput; + if (scenario === 'hardlink') linkSync(resultingInput, samplesPath); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, samplesPath, 'transition-changed', resultingInput, + ), { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + scenario === 'identical' + ? 'Changed-transition benchmark inputs must differ.' + : scenario === 'overwrite' || scenario === 'hardlink' + ? 'Revision benchmark output must not overwrite its input.' + : 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(root); + if (scenario === 'overwrite' || scenario === 'hardlink') { + expect(readFileSync(resultingInput, 'utf8')).toBe('private resulting source'); + } else expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each(['transition-changed', 'transition', 'canonicalization', undefined] as const)( + 'requires a resulting input only for changed transitions (%s)', (operation) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-arguments-')); + const input = join(root, 'input.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(modulePath, 'throw new Error("must not execute");'); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, output, operation, + operation === 'transition-changed' ? undefined : input, + ), { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stderr.trim()).toBe(operation === undefined + ? 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ' + : 'Only changed-transition measurement requires a resulting input.'); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + it('isolates strict envelope canonicalization from digest-provider cost', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-canonicalization-measurement-')); const input = join(root, 'large.json'); diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts index 92fa1c34..f4404c8a 100644 --- a/src/performanceWorkflowContract.test.ts +++ b/src/performanceWorkflowContract.test.ts @@ -52,6 +52,9 @@ describe('performance evidence workflow contract', () => { '--revision-input "$corpus_dir/${{ matrix.profile }}.envelope.json"', ); expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); + expect(workflow).toContain( + '--resulting-input "$corpus_dir/${{ matrix.profile }}.changed.envelope.json"', + ); expect(workflow).toContain('--profile "${{ matrix.profile }}"'); expect(workflow).toContain('--samples "${{ matrix.samples }}"'); expect(workflow).toContain( From 03345b029624f5c3945ebeec5d075db14d259b6e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 16:53:51 +0900 Subject: [PATCH 247/260] test(benchmarks): reject unmeasured operation invocations Signed-off-by: Seongho Bae --- src/performanceAutosaveMeasurementContract.test.ts | 10 ++++++++-- src/performanceHtmlSerializationMeasurement.test.ts | 3 ++- src/performanceMarkdownMeasurementContract.test.ts | 2 +- src/performanceRevisionMeasurementContract.test.ts | 13 +++++++++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index fca9984e..5ceb60ca 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -39,7 +39,9 @@ describe('autosave enqueue performance measurement', () => { const inputPath = join(directory, 'document-envelope.json'); const outputPath = join(directory, 'samples.json'); const moduleSource = [ + 'let queueCount = 0;', 'export function createDocumentAutosaveQueue(options) {', + " if (++queueCount > 2) throw new Error('Unmeasured invocation');", ' return Object.freeze({', ' async enqueue(evidence) {', " if (evidence.envelope.documentJson.content[0].content[0].text !== 'profile-bound synthetic input') throw new Error('wrong profile input');", @@ -169,7 +171,9 @@ describe('autosave enqueue performance measurement', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-coalescing-')); const modulePath = join(directory, 'autosave.mjs'); const outputPath = join(directory, 'samples.json'); - const moduleSource = `export function createDocumentAutosaveQueue(options) { + const moduleSource = `let queueCount = 0; +export function createDocumentAutosaveQueue(options) { + if (++queueCount > 2) throw new Error('Unmeasured invocation'); let active; return { enqueue(evidence) { @@ -214,7 +218,9 @@ describe('autosave enqueue performance measurement', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-commit-')); const modulePath = join(directory, 'autosave.mjs'); const outputPath = join(directory, 'samples.json'); - const moduleSource = `export function createDocumentAutosaveQueue(options) { + const moduleSource = `let queueCount = 0; +export function createDocumentAutosaveQueue(options) { + if (++queueCount > 2) throw new Error('Unmeasured invocation'); return { enqueue(evidence) { return Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts index 7eaff65e..7a870dda 100644 --- a/src/performanceHtmlSerializationMeasurement.test.ts +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -42,8 +42,9 @@ describe('HTML serialization performance measurement', () => { writeFileSync( modulePath, [ + 'let measuredCalls = 0;', "export function markdownToHtml() { throw new Error('wrong serialization direction'); }", - "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + "export function htmlToMarkdown(source) { if (++measuredCalls > 2) throw new Error('Unmeasured invocation'); return source.replace(/<[^>]+>/gu, '').trim(); }", '', ].join('\n'), 'utf8', diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 0ec5fdc2..01c5c44c 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -90,7 +90,7 @@ describe('Markdown runtime measurement contract', () => { writeFileSync(input, '# Buyer benchmark fixture\n\nSynthetic content only.\n', 'utf8'); writeFileSync( modulePath, - "export function markdownToHtml(source) { return `

${source.length}

`; }\n", + "let measuredCalls = 0;\nexport function markdownToHtml(source) { if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); return `

${source.length}

`; }\n", 'utf8', ); const artifactSha256 = fileSha256(modulePath); diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 17b09299..1b1db3b6 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -110,7 +110,7 @@ describe('revision-evidence runtime measurement contract', () => { writeSyntheticEnvelope(input); writeFileSync( modulePath, - 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'let measuredCalls = 0;\nexport async function createDocumentEnvelopeRevisionEvidenceBytes(source) { if (++measuredCalls > 3) throw new Error("Unmeasured invocation"); return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', 'utf8', ); @@ -168,8 +168,9 @@ describe('revision-evidence runtime measurement contract', () => { writeSyntheticEnvelope(input); writeFileSync( modulePath, - `const revision = { digestHex: '${'d'.repeat(64)}' }; -export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, + `let measuredCalls = 0; +const revision = { digestHex: '${'d'.repeat(64)}' }; +export async function createDocumentEnvelopeTransitionEvidenceBytes() { if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, 'utf8', ); @@ -210,7 +211,9 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { 'Synthetic benchmark content', 'Synthetic benchmark content with an edit', )); writeFileSync(modulePath, ` +let measuredCalls = 0; export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, resulting) { + if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); if (previous.toString() !== ${JSON.stringify(readFileSync(input, 'utf8'))}) throw new Error('wrong previous source'); if (resulting.toString() !== ${JSON.stringify(readFileSync(operation === 'transition' ? input : resultingInput, 'utf8'))}) throw new Error('wrong resulting source'); return { previousRevision: { digestHex: '${'a'.repeat(64)}' }, resultingRevision: { digestHex: '${(equalDigests ? 'a' : 'b').repeat(64)}' }, changed: ${changed} }; @@ -314,7 +317,9 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, re writeSyntheticEnvelope(input); writeFileSync( modulePath, - `export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + `let measuredCalls = 0; +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); const digest = await provider.digest('SHA-256', source); return { revision: { digestHex: Buffer.from(digest).toString('hex') } }; }\n`, From 8f1b389187995d2eda992fc429e2e2a9bfddfd60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 17:00:45 +0900 Subject: [PATCH 248/260] fix(benchmarks): record the first operation invocation Signed-off-by: Seongho Bae --- benchmarks/README.md | 15 ++++++++++ benchmarks/measure-autosave.mjs | 1 - benchmarks/measure-markdown.mjs | 18 +++-------- benchmarks/measure-revision-evidence.mjs | 2 -- docs/TRD.md | 2 ++ docs/product-technical-gap-baseline.md | 4 ++- ...ormanceMarkdownMeasurementContract.test.ts | 30 +++++++++++++++++++ 7 files changed, 54 insertions(+), 18 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 8e87bff0..f0eafeea 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,6 +7,21 @@ buyer-workload performance, real-device input behavior, or the 20 ms target. Envelope fixtures contain plain paragraphs; Markdown list, table, and image syntax inside those paragraphs is not a rich editor document tree. +## First-invocation accounting + +The revision, Markdown/HTML, and autosave latency producers record the first +operation invocation and every subsequent requested sample, without an +unrecorded warmup invocation. Each result must pass its scenario checks before +samples can be published. Module loading and input preparation remain outside +the timer; autosave queue setup and coalescing-scenario preparation retain +their existing timer boundaries. This is not process-startup latency. + +Samples recorded before this change retain their original meaning. Start a +new baseline for the new measurement method; a difference between those +generations is not evidence of a product speedup. Keep each generation's raw +samples and exact source revision. Office rendering and the separate memory +settling analysis are unchanged. + ## Transition scenarios | Operation | Inputs | Required result | Metric prefix | diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index e2ca1295..179f329f 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -548,7 +548,6 @@ async function main() { coalescing: measureOneCoalescing, commit: measureOneCommit, }[args.operation]; - await measure(measuredModule.createDocumentAutosaveQueue, evidence); const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { samples.push( diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 049b1df2..09209f3b 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -425,15 +425,6 @@ async function main() { ); } - const warmup = runMeasuredSerialization( - serializer, - source, - contract.executionFailure, - ); - if (typeof warmup !== 'string') { - throw new Error(contract.returnFailure); - } - const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); @@ -443,11 +434,10 @@ async function main() { contract.executionFailure, ); const elapsed = performance.now() - start; - if ( - typeof output !== 'string' || - !Number.isFinite(elapsed) || - elapsed < 0 - ) { + if (typeof output !== 'string') { + throw new Error(contract.returnFailure); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { throw new Error('Markdown measurement produced invalid runtime evidence.'); } samples.push(elapsed); diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 8c20e302..c953a670 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -461,8 +461,6 @@ async function main() { ); } - await runMeasuredEvidence(createEvidence, source, resultingSource, args.operation); - const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); diff --git a/docs/TRD.md b/docs/TRD.md index 80cec07a..cb9a9335 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -133,6 +133,8 @@ Release publication verifies one exact integrated protected source head, expecte Queued, cancelled, skipped-required, absent, stale-head, predecessor-head, status-only, author-only, or synthetic-merge evidence is not success. A commit status, automated model verdict, comment, formal review, merge authority, and external registry publication are distinct evidence classes. +Active PR performance research records the first operation invocation without an unrecorded latency warmup, with timing boundaries and generation-comparison limits documented in [`benchmarks/README.md`](../benchmarks/README.md). This harness does not establish buyer-workload latency or a supported document-size guarantee; a measurement-method change starts a new baseline rather than proving a product speedup. + ## Security, privacy, and operability dependencies `SECURITY.md`, `docs/THREAT_MODEL.md`, `docs/TEST_STRATEGY.md`, `docs/OPERABILITY.md`, `docs/TRACEABILITY.md`, and the detailed ADR corpus are part of this technical contract. Root `SECURITY.md` is `implemented_on_protected_main` and is the normative private vulnerability-reporting/coordinated-disclosure policy. ADR 0017 records the durable decision, ownership boundary, claim limits, and recovery/supersession semantics without duplicating the policy text. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 18fd8749..e51b4751 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -129,7 +129,9 @@ After P0 blockers and independent non-conflicting lanes permit, prioritize: 1. provider-neutral comments, suggestions, review targets, and revision-scoped proposal acceptance without moving identity or persistence into Inkspan; 2. measured large-document latency/memory support envelopes with realistic - fixtures and deterministic failure behavior; + fixtures and deterministic failure behavior; the active research harness's + [first-invocation accounting and claim limits](../benchmarks/README.md) + do not close this buyer-workload gap; 3. CJK IME, touch, and mobile editing assurance with truthful real-device versus emulated support claims; 4. an executable packed-package reference host proving SSR/hydration, native diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 01c5c44c..aca6ca19 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -141,6 +141,36 @@ describe('Markdown runtime measurement contract', () => { } }); + it.each([ + ['markdown-to-html', 'markdownToHtml', 1], + ['markdown-to-html', 'markdownToHtml', 3], + ['html-to-markdown', 'htmlToMarkdown', 1], + ['html-to-markdown', 'htmlToMarkdown', 3], + ] as const)('rejects %s non-string result from %s on invocation %i', (operation, exportName, invalidCall) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-serialization-invalid-result-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, + `let calls = 0;\nexport function ${exportName}() { return ++calls === ${invalidCall} ? { privateContent: 'never publish this' } : ''; }\n`, + 'utf8'); + const args = measurementArguments(input, modulePath, samplesPath); + args.splice(5, 0, '--operation', operation); + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(`Measured ${exportName}() must return a string.`); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed before output when the measured module lacks the public serializer', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); const input = join(root, 'small.md'); From b6d83403c530f8a7a0d0c45cfbe8954b1d9c849a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 19:01:34 +0900 Subject: [PATCH 249/260] test(benchmarks): distinguish measurement generations Signed-off-by: Seongho Bae --- ...ormanceAutosaveMeasurementContract.test.ts | 2 +- ...rmanceHtmlSerializationMeasurement.test.ts | 2 ++ ...ormanceMarkdownMeasurementContract.test.ts | 5 ++-- ...manceMeasurementStatisticsContract.test.ts | 30 +++++++++++++++---- ...rmanceRegressionComparatorContract.test.ts | 28 +++++++++++++---- ...ormanceRevisionMeasurementContract.test.ts | 4 +-- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index 5ceb60ca..454912c0 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -131,7 +131,7 @@ describe('autosave enqueue performance measurement', () => { samples?: unknown[]; }; expect(evidence).toMatchObject({ - contractVersion: 1, + contractVersion: 2, benchmarkId: 'autosave-enqueue-small', unit: 'ms', sourceCommitSha, diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts index 7a870dda..0961dbec 100644 --- a/src/performanceHtmlSerializationMeasurement.test.ts +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -82,11 +82,13 @@ describe('HTML serialization performance measurement', () => { expect(result.stderr).toBe(''); const evidence = JSON.parse(readFileSync(output, 'utf8')) as { + contractVersion: number; benchmarkId: string; unit: string; documentProfile: string; samples: unknown[]; }; + expect(evidence.contractVersion).toBe(2); expect(evidence.benchmarkId).toBe('html-serialization-small'); expect(evidence.unit).toBe('ms'); expect(evidence.documentProfile).toBe('small'); diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index aca6ca19..eec64541 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -14,7 +14,7 @@ import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSamples { - readonly contractVersion: 1; + readonly contractVersion: 2; readonly benchmarkId: string; readonly unit: 'ms'; readonly sourceCommitSha: string; @@ -105,7 +105,7 @@ describe('Markdown runtime measurement contract', () => { readFileSync(samplesPath, 'utf8'), ) as BenchmarkSamples; expect(samples).toMatchObject({ - contractVersion: 1, + contractVersion: 2, benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, @@ -132,6 +132,7 @@ describe('Markdown runtime measurement contract', () => { readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), ) as { sampleCount: number; benchmarkId: string; unit: string }; expect(summary).toMatchObject({ + contractVersion: 2, sampleCount: 3, benchmarkId: 'markdown-serialization-large', unit: 'ms', diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 9a4646d4..2843dfee 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -16,7 +16,7 @@ import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSummary { - readonly contractVersion: 1; + readonly contractVersion: 1 | 2; readonly benchmarkId: string; readonly unit: string; readonly sourceCommitSha: string; @@ -37,12 +37,12 @@ const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); -function writeInput(path: string, samples: readonly number[]): void { +function writeInput(path: string, samples: readonly number[], contractVersion: unknown = 1): void { writeFileSync( path, `${JSON.stringify( { - contractVersion: 1, + contractVersion, benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, @@ -74,19 +74,19 @@ function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummar } describe('deterministic benchmark sample statistics', () => { - it('writes reproducible nearest-rank JSON and human-readable summaries with provenance metadata', () => { + it.each([1, 2] as const)('preserves generation %i in reproducible JSON and human-readable summaries', (contractVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); const input = join(root, 'samples.json'); const first = join(root, 'first'); const second = join(root, 'second'); try { - writeInput(input, [20, 10, 40, 30, 50]); + writeInput(input, [20, 10, 40, 30, 50], contractVersion); const firstSummary = runSummary(input, first); const secondSummary = runSummary(input, second); expect(firstSummary).toEqual(secondSummary); expect(firstSummary).toEqual({ - contractVersion: 1, + contractVersion, benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, @@ -104,6 +104,7 @@ describe('deterministic benchmark sample statistics', () => { }); const expectedText = [ + `contract_version=${contractVersion}`, 'benchmark=markdown-serialization-large', 'unit=ms', `source_commit_sha=${SOURCE_COMMIT_SHA}`, @@ -127,6 +128,23 @@ describe('deterministic benchmark sample statistics', () => { } }); + it.each([0, 3, '2', null])('rejects unsupported measurement generation %s', (contractVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-generation-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], contractVersion); + const result = spawnSync(process.execPath, + [script, '--input', input, '--output', output], { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark sample contractVersion must be 1 or 2.'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed when immutable provenance metadata is missing', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); const input = join(root, 'samples.json'); diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index e98860fc..cc415ba3 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -10,6 +10,7 @@ const ARTIFACT_SHA256 = 'b'.repeat(64); const CURRENT_ARTIFACT_SHA256 = 'c'.repeat(64); type SummaryOverrides = Partial<{ + contractVersion: number; benchmarkId: string; unit: string; sourceCommitSha: string; @@ -75,21 +76,24 @@ function runComparison( } describe('benchmark regression comparator contract', () => { - it.each(['editor-input-large', 'transition-changed-evidence-large'])( - 'compares only exact-context %s evidence with an explicit tolerance', (benchmarkId) => { + it.each([ + ['editor-input-large', 1], ['editor-input-large', 2], + ['transition-changed-evidence-large', 1], ['transition-changed-evidence-large', 2], + ] as const)( + 'compares exact-context %s generation %i evidence with an explicit tolerance', (benchmarkId, contractVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); try { const result = runComparison( root, - summary({ benchmarkId }), - summary({ benchmarkId, artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + summary({ benchmarkId, contractVersion }), + summary({ benchmarkId, contractVersion, artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), '5', ); expect(result.status).toBe(0); expect(result.stderr).toBe(''); expect(JSON.parse(result.stdout)).toEqual({ - contractVersion: 1, + contractVersion, benchmarkId, unit: 'ms', documentProfile: 'large', @@ -113,6 +117,20 @@ describe('benchmark regression comparator contract', () => { } }); + it.each([[1, 2], [2, 1]])('rejects generation %i versus %i even with a generous tolerance', (baselineVersion, currentVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-generation-')); + try { + const result = runComparison(root, + summary({ contractVersion: baselineVersion }), + summary({ contractVersion: currentVersion, artifactSha256: CURRENT_ARTIFACT_SHA256 }), '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summaries are not comparable: contractVersion differs.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('rejects comparisons between changed and unchanged transition scenarios', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-compare-')); try { diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 1b1db3b6..1599f882 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -14,7 +14,7 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; interface BenchmarkSamples { - readonly contractVersion: 1; + readonly contractVersion: 2; readonly benchmarkId: string; readonly unit: 'ms'; readonly sourceCommitSha: string; @@ -123,7 +123,7 @@ describe('revision-evidence runtime measurement contract', () => { readFileSync(samplesPath, 'utf8'), ) as BenchmarkSamples; expect(samples).toMatchObject({ - contractVersion: 1, + contractVersion: 2, benchmarkId: 'revision-evidence-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, From e74a8a73db8bbc6c9946b9dec04a59e866255179 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 19:03:06 +0900 Subject: [PATCH 250/260] fix(benchmarks): reject cross-generation latency comparisons Signed-off-by: Seongho Bae --- benchmarks/README.md | 11 +++++++++++ benchmarks/compare-summaries.mjs | 8 +++++--- benchmarks/measure-autosave.mjs | 2 +- benchmarks/measure-markdown.mjs | 2 +- benchmarks/measure-revision-evidence.mjs | 2 +- benchmarks/summarize-samples.mjs | 8 +++++--- docs/TRD.md | 2 ++ 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index f0eafeea..92907384 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,6 +22,17 @@ generations is not evidence of a product speedup. Keep each generation's raw samples and exact source revision. Office rendering and the separate memory settling analysis are unchanged. +New JavaScript latency samples use `contractVersion: 2` to identify this +first-invocation method. The summarizer preserves that version in JSON and +prints `contract_version` in its text receipt. Version 1 inputs remain readable +as legacy evidence, including Office output, but the comparator rejects a +version 1 / version 2 pair before calculating any improvement or regression. +Older version-1-only consumers reject new samples; update the evidence tools +together. Do not relabel historical samples or assume their invocation +accounting. The suite inventory and corpus locks retain their independent +version 1 contracts; this version change does not alter their shapes or the +published editor API. + ## Transition scenarios | Operation | Inputs | Required result | Metric prefix | diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index b75f9d0b..ba83e519 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -44,6 +44,7 @@ const SUMMARY_KEYS = new Set([ 'maximum', ]); const COMPARABLE_FIELDS = [ + 'contractVersion', 'benchmarkId', 'unit', 'documentProfile', @@ -165,8 +166,8 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary input has an unsupported shape.'); } - if (value.contractVersion !== 1) { - throw new Error('Benchmark summary contractVersion must be 1.'); + if (value.contractVersion !== 1 && value.contractVersion !== 2) { + throw new Error('Benchmark summary contractVersion must be 1 or 2.'); } if ( typeof value.benchmarkId !== 'string' || @@ -247,6 +248,7 @@ function validateSummary(value) { } return Object.freeze({ + contractVersion: value.contractVersion, benchmarkId: value.benchmarkId, unit: value.unit, sourceCommitSha: value.sourceCommitSha, @@ -301,7 +303,7 @@ function compare(baseline, current, metric, maxRegressionPercent) { ); } return Object.freeze({ - contractVersion: 1, + contractVersion: baseline.contractVersion, benchmarkId: baseline.benchmarkId, unit: baseline.unit, documentProfile: baseline.documentProfile, diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 179f329f..bf3502a0 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -571,7 +571,7 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 1, + contractVersion: 2, benchmarkId: `autosave-${args.operation}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 09209f3b..3bbbb497 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -460,7 +460,7 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 1, + contractVersion: 2, benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index c953a670..b739cd66 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -489,7 +489,7 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 1, + contractVersion: 2, benchmarkId: args.operation === 'canonicalization' ? `envelope-canonicalization-${args.profile}` diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 80eb03c9..ef93d728 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -137,8 +137,8 @@ function validateInput(value) { if (Object.keys(value).some((key) => !BENCHMARK_INPUT_KEYS.has(key))) { throw new Error('Benchmark sample input contains unsupported fields.'); } - if (value.contractVersion !== 1) { - throw new Error('Benchmark sample contractVersion must be 1.'); + if (value.contractVersion !== 1 && value.contractVersion !== 2) { + throw new Error('Benchmark sample contractVersion must be 1 or 2.'); } if ( typeof value.benchmarkId !== 'string' || @@ -202,6 +202,7 @@ function validateInput(value) { throw new Error('Benchmark samples must be finite non-negative numbers.'); } return Object.freeze({ + contractVersion: value.contractVersion, benchmarkId: value.benchmarkId, unit: value.unit, sourceCommitSha: value.sourceCommitSha, @@ -221,7 +222,7 @@ function nearestRank(sorted, percentile) { function summarize(input) { const sorted = [...input.samples].sort((left, right) => left - right); return Object.freeze({ - contractVersion: 1, + contractVersion: input.contractVersion, benchmarkId: input.benchmarkId, unit: input.unit, sourceCommitSha: input.sourceCommitSha, @@ -241,6 +242,7 @@ function summarize(input) { function formatSummary(summary) { return [ + `contract_version=${summary.contractVersion}`, `benchmark=${summary.benchmarkId}`, `unit=${summary.unit}`, `source_commit_sha=${summary.sourceCommitSha}`, diff --git a/docs/TRD.md b/docs/TRD.md index cb9a9335..ee8f0424 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -135,6 +135,8 @@ Queued, cancelled, skipped-required, absent, stale-head, predecessor-head, statu Active PR performance research records the first operation invocation without an unrecorded latency warmup, with timing boundaries and generation-comparison limits documented in [`benchmarks/README.md`](../benchmarks/README.md). This harness does not establish buyer-workload latency or a supported document-size guarantee; a measurement-method change starts a new baseline rather than proving a product speedup. +The active research sample/summary contract marks first-invocation JavaScript latency evidence as version 2. Legacy version 1 remains readable, but cross-version comparisons fail closed. Corpus and suite-inventory versions are independent; no editor API or supported-performance promise changes. + ## Security, privacy, and operability dependencies `SECURITY.md`, `docs/THREAT_MODEL.md`, `docs/TEST_STRATEGY.md`, `docs/OPERABILITY.md`, `docs/TRACEABILITY.md`, and the detailed ADR corpus are part of this technical contract. Root `SECURITY.md` is `implemented_on_protected_main` and is the normative private vulnerability-reporting/coordinated-disclosure policy. ADR 0017 records the durable decision, ownership boundary, claim limits, and recovery/supersession semantics without duplicating the policy text. From 43e946aa77895dc54345361792f523af3840c95f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:02:33 +0900 Subject: [PATCH 251/260] test: require document identity in Markdown measurements Signed-off-by: Seongho Bae --- src/performanceMarkdownMeasurementContract.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index eec64541..c3c6467a 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -110,6 +110,7 @@ describe('Markdown runtime measurement contract', () => { unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, artifactSha256, + inputSha256: fileSha256(input), documentProfile: 'large', runtimeId: RUNTIME_ID, referenceHardwareId: HARDWARE_ID, @@ -136,6 +137,7 @@ describe('Markdown runtime measurement contract', () => { sampleCount: 3, benchmarkId: 'markdown-serialization-large', unit: 'ms', + inputSha256: fileSha256(input), }); } finally { rmSync(root, { recursive: true, force: true }); From 69bdcc5c4d8ba757fb84d6e96c77fb2704cecf7c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:14:11 +0900 Subject: [PATCH 252/260] test: bind revision and autosave measurements to their inputs Signed-off-by: Seongho Bae --- src/performanceAutosaveMeasurementContract.test.ts | 2 ++ src/performanceRevisionMeasurementContract.test.ts | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index 454912c0..91afa593 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -136,6 +136,7 @@ describe('autosave enqueue performance measurement', () => { unit: 'ms', sourceCommitSha, artifactSha256: sha256(moduleSource), + inputSha256: sha256(readFileSync(inputPath, 'utf8')), documentProfile: 'small', runtimeId, referenceHardwareId, @@ -146,6 +147,7 @@ describe('autosave enqueue performance measurement', () => { 'benchmarkId', 'contractVersion', 'documentProfile', + 'inputSha256', 'referenceHardwareId', 'runtimeId', 'samples', diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 1599f882..864f1333 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -128,6 +128,7 @@ describe('revision-evidence runtime measurement contract', () => { unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), documentProfile: 'large', runtimeId: RUNTIME_ID, referenceHardwareId: HARDWARE_ID, @@ -153,6 +154,7 @@ describe('revision-evidence runtime measurement contract', () => { sampleCount: 3, benchmarkId: 'revision-evidence-large', unit: 'ms', + inputSha256: fileSha256(input), }); } finally { rmSync(root, { recursive: true, force: true }); @@ -183,6 +185,7 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes() { if (++me expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ benchmarkId: 'transition-evidence-large', artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), samples: expect.any(Array), }); } finally { @@ -230,6 +233,8 @@ export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, re const output = readFileSync(samplesPath, 'utf8'); expect(JSON.parse(output)).toMatchObject({ benchmarkId: 'transition-changed-evidence-large', + inputSha256: fileSha256(input), + resultingInputSha256: fileSha256(resultingInput), samples: expect.any(Array), }); expect(JSON.parse(output).samples).toHaveLength(3); @@ -335,6 +340,7 @@ export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ benchmarkId: 'envelope-canonicalization-large', artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), samples: expect.any(Array), }); } finally { From ce48badc22ecf45cec8ff3f14e40a323deaae0fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:24:27 +0900 Subject: [PATCH 253/260] test(performance): reject mismatched measurement input identities Signed-off-by: Seongho Bae --- ...manceMeasurementStatisticsContract.test.ts | 50 +++++++++++++++++-- ...rmanceRegressionComparatorContract.test.ts | 49 +++++++++++++++++- 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 2843dfee..786e93f6 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -16,7 +16,7 @@ import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSummary { - readonly contractVersion: 1 | 2; + readonly contractVersion: 1 | 2 | 3; readonly benchmarkId: string; readonly unit: string; readonly sourceCommitSha: string; @@ -47,6 +47,7 @@ function writeInput(path: string, samples: readonly number[], contractVersion: u unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, artifactSha256: ARTIFACT_SHA256, + ...(contractVersion === 3 ? { inputSha256: 'd'.repeat(64) } : {}), documentProfile: 'large', runtimeId: 'node-22.18.0', referenceHardwareId: 'github-actions-ubuntu-24.04-x64', @@ -74,7 +75,7 @@ function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummar } describe('deterministic benchmark sample statistics', () => { - it.each([1, 2] as const)('preserves generation %i in reproducible JSON and human-readable summaries', (contractVersion) => { + it.each([1, 2, 3] as const)('preserves generation %i in reproducible JSON and human-readable summaries', (contractVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); const input = join(root, 'samples.json'); const first = join(root, 'first'); @@ -91,6 +92,7 @@ describe('deterministic benchmark sample statistics', () => { unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, artifactSha256: ARTIFACT_SHA256, + ...(contractVersion === 3 ? { inputSha256: 'd'.repeat(64) } : {}), documentProfile: 'large', runtimeId: 'node-22.18.0', referenceHardwareId: 'github-actions-ubuntu-24.04-x64', @@ -109,6 +111,7 @@ describe('deterministic benchmark sample statistics', () => { 'unit=ms', `source_commit_sha=${SOURCE_COMMIT_SHA}`, `artifact_sha256=${ARTIFACT_SHA256}`, + ...(contractVersion === 3 ? [`input_sha256=${'d'.repeat(64)}`] : []), 'document_profile=large', 'runtime_id=node-22.18.0', 'reference_hardware_id=github-actions-ubuntu-24.04-x64', @@ -128,7 +131,7 @@ describe('deterministic benchmark sample statistics', () => { } }); - it.each([0, 3, '2', null])('rejects unsupported measurement generation %s', (contractVersion) => { + it.each([0, 4, '2', null])('rejects unsupported measurement generation %s', (contractVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-generation-')); const input = join(root, 'samples.json'); const output = join(root, 'output'); @@ -138,13 +141,52 @@ describe('deterministic benchmark sample statistics', () => { [script, '--input', input, '--output', output], { encoding: 'utf8' }); expect(result.status).toBe(1); expect(result.stdout).toBe(''); - expect(result.stderr.trim()).toBe('Benchmark sample contractVersion must be 1 or 2.'); + expect(result.stderr.trim()).toBe('Benchmark sample contractVersion must be 1, 2 or 3.'); expect(existsSync(join(output, 'summary.json'))).toBe(false); } finally { rmSync(root, { recursive: true, force: true }); } }); + it('preserves ordered changed-transition input identities in both receipts', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-inputs-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], 3); + const samples = JSON.parse(readFileSync(input, 'utf8')); + samples.benchmarkId = 'transition-changed-evidence-large'; + samples.resultingInputSha256 = 'e'.repeat(64); + writeFileSync(input, JSON.stringify(samples)); + expect(runSummary(input, output)).toMatchObject({ inputSha256: 'd'.repeat(64), resultingInputSha256: 'e'.repeat(64) }); + expect(readFileSync(join(output, 'summary.txt'), 'utf8')).toContain(`input_sha256=${'d'.repeat(64)}\nresulting_input_sha256=${'e'.repeat(64)}\n`); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { inputSha256: undefined }, { inputSha256: 'private-path' }, { inputSha256: 'A'.repeat(64) }, + { resultingInputSha256: 'e'.repeat(64) }, { contractVersion: 2 }, + { benchmarkId: 'transition-changed-evidence-large' }, + { benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: 'd'.repeat(64) }, + ])('rejects invalid input identity metadata %j without writing evidence', (overrides) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-input-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], 3); + writeFileSync(input, JSON.stringify({ ...JSON.parse(readFileSync(input, 'utf8')), ...overrides })); + const result = spawnSync(process.execPath, [script, '--input', input, '--output', output], { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).not.toContain('private-path'); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed when immutable provenance metadata is missing', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); const input = join(root, 'samples.json'); diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index cc415ba3..ed2ef701 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -15,6 +15,8 @@ type SummaryOverrides = Partial<{ unit: string; sourceCommitSha: string; artifactSha256: string; + inputSha256: string; + resultingInputSha256: string; documentProfile: string; runtimeId: string; referenceHardwareId: string; @@ -44,6 +46,11 @@ function summary(overrides: SummaryOverrides = {}) { p75: 90, p95: 100, maximum: 110, + ...(overrides.contractVersion === 3 ? { + inputSha256: 'd'.repeat(64), + ...(overrides.benchmarkId?.startsWith('transition-changed-evidence-') + ? { resultingInputSha256: 'e'.repeat(64) } : {}), + } : {}), ...overrides, }; } @@ -79,6 +86,7 @@ describe('benchmark regression comparator contract', () => { it.each([ ['editor-input-large', 1], ['editor-input-large', 2], ['transition-changed-evidence-large', 1], ['transition-changed-evidence-large', 2], + ['editor-input-large', 3], ['transition-changed-evidence-large', 3], ] as const)( 'compares exact-context %s generation %i evidence with an explicit tolerance', (benchmarkId, contractVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); @@ -96,6 +104,11 @@ describe('benchmark regression comparator contract', () => { contractVersion, benchmarkId, unit: 'ms', + ...(contractVersion === 3 ? { + inputSha256: 'd'.repeat(64), + ...(benchmarkId.startsWith('transition-changed-evidence-') + ? { resultingInputSha256: 'e'.repeat(64) } : {}), + } : {}), documentProfile: 'large', runtimeId: 'chromium-1.62.0', referenceHardwareId: 'github-actions-ubuntu-24.04-x64', @@ -117,7 +130,7 @@ describe('benchmark regression comparator contract', () => { } }); - it.each([[1, 2], [2, 1]])('rejects generation %i versus %i even with a generous tolerance', (baselineVersion, currentVersion) => { + it.each([[1, 2], [2, 1], [2, 3], [3, 2], [1, 3], [3, 1]])('rejects generation %i versus %i even with a generous tolerance', (baselineVersion, currentVersion) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-generation-')); try { const result = runComparison(root, @@ -131,6 +144,40 @@ describe('benchmark regression comparator contract', () => { } }); + it.each(['inputSha256', 'resultingInputSha256'] as const)('rejects a changed %s before reporting a speedup', (field) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-input-')); + try { + const baseline = summary({ contractVersion: 3, benchmarkId: 'transition-changed-evidence-large' }); + const current = { ...baseline, artifactSha256: CURRENT_ARTIFACT_SHA256, [field]: 'f'.repeat(64), p95: 90 }; + const result = runComparison(root, baseline, current, '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(`Benchmark summaries are not comparable: ${field} differs.`); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { contractVersion: 3, inputSha256: undefined }, + { contractVersion: 3, inputSha256: 'private-path' }, + { contractVersion: 3, inputSha256: 'A'.repeat(64) }, + { contractVersion: 3, resultingInputSha256: 'e'.repeat(64) }, + { contractVersion: 2, inputSha256: 'd'.repeat(64) }, + { contractVersion: 3, benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: undefined }, + { contractVersion: 3, benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: 'd'.repeat(64) }, + ])('rejects invalid input identity metadata %j', (overrides) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-input-invalid-')); + try { + const result = runComparison(root, summary(overrides), summary({ ...overrides, artifactSha256: CURRENT_ARTIFACT_SHA256 }), '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).not.toContain('private-path'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('rejects comparisons between changed and unchanged transition scenarios', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-compare-')); try { From c4fba276b1ea8725a61f354bc0c3e26c58aa6c54 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:29:18 +0900 Subject: [PATCH 254/260] fix(performance): bind measurement comparisons to captured input bytes Signed-off-by: Seongho Bae --- benchmarks/README.md | 27 ++++++++------ benchmarks/compare-summaries.mjs | 36 ++++++++++++++----- benchmarks/measure-autosave.mjs | 15 +++++--- benchmarks/measure-markdown.mjs | 10 ++++-- benchmarks/measure-revision-evidence.mjs | 8 ++++- benchmarks/summarize-samples.mjs | 31 +++++++++++++--- docs/TRD.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- ...ormanceAutosaveMeasurementContract.test.ts | 2 +- ...rmanceHtmlSerializationMeasurement.test.ts | 4 ++- ...ormanceMarkdownMeasurementContract.test.ts | 6 ++-- ...manceMeasurementStatisticsContract.test.ts | 3 +- ...ormanceRevisionMeasurementContract.test.ts | 4 +-- 13 files changed, 108 insertions(+), 42 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 92907384..71e8676c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,16 +22,23 @@ generations is not evidence of a product speedup. Keep each generation's raw samples and exact source revision. Office rendering and the separate memory settling analysis are unchanged. -New JavaScript latency samples use `contractVersion: 2` to identify this -first-invocation method. The summarizer preserves that version in JSON and -prints `contract_version` in its text receipt. Version 1 inputs remain readable -as legacy evidence, including Office output, but the comparator rejects a -version 1 / version 2 pair before calculating any improvement or regression. -Older version-1-only consumers reject new samples; update the evidence tools -together. Do not relabel historical samples or assume their invocation -accounting. The suite inventory and corpus locks retain their independent -version 1 contracts; this version change does not alter their shapes or the -published editor API. +Version 2 introduced this first-invocation method. New JavaScript latency samples +use `contractVersion: 3` and additionally identify the captured input bytes with +`inputSha256`. The digest is derived before measurement from the same bounded +read used by the operation, not a caller-supplied claim or a later file read. +Changed transitions also record `resultingInputSha256` in its distinct resulting +role. Autosave with no input file hashes the UTF-8 JSON representation of its +fixed synthetic envelope; that input remains synthetic regardless of profile. +Input digests are workload identifiers, not anonymization or authenticity proofs. + +The summarizer preserves these identities in JSON and text. The comparator +rejects different versions or input identities before calculating a verdict, +even when document profiles match. Versions 1 and 2 remain readable with their +original meanings; Office output remains version 1. Do not backfill identities, +relabel historical samples, or present a cross-generation difference as a +speedup. Update producers and evidence readers together. The suite inventory and +corpus locks retain their independent version 1 contracts; published editor APIs +and timer boundaries are unchanged. This does not establish a real-world corpus. ## Transition scenarios diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index ba83e519..eda882a7 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -52,6 +52,8 @@ const COMPARABLE_FIELDS = [ 'referenceHardwareId', 'sampleCount', 'percentileMethod', + 'inputSha256', + 'resultingInputSha256', ]; const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); @@ -159,15 +161,8 @@ function validateSummary(value) { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Benchmark summary input must be an object.'); } - const keys = Object.keys(value); - if ( - keys.length !== SUMMARY_KEYS.size || - keys.some((key) => !SUMMARY_KEYS.has(key)) - ) { - throw new Error('Benchmark summary input has an unsupported shape.'); - } - if (value.contractVersion !== 1 && value.contractVersion !== 2) { - throw new Error('Benchmark summary contractVersion must be 1 or 2.'); + if (![1, 2, 3].includes(value.contractVersion)) { + throw new Error('Benchmark summary contractVersion must be 1, 2 or 3.'); } if ( typeof value.benchmarkId !== 'string' || @@ -175,6 +170,26 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary benchmarkId is invalid.'); } + const inputIdentity = {}; + if (value.contractVersion === 3) { + inputIdentity.inputSha256 = value.inputSha256; + if (value.benchmarkId.startsWith('transition-changed-evidence-')) { + inputIdentity.resultingInputSha256 = value.resultingInputSha256; + if (value.inputSha256 === value.resultingInputSha256) { + throw new Error('Benchmark summary changed-transition inputs must differ.'); + } + } + for (const digest of Object.values(inputIdentity)) { + if (typeof digest !== 'string' || !SHA256_PATTERN.test(digest)) { + throw new Error('Benchmark summary input identities must be lowercase SHA-256 digests.'); + } + } + } + const keys = Object.keys(value); + const allowedKeys = new Set([...SUMMARY_KEYS, ...Object.keys(inputIdentity)]); + if (keys.length !== allowedKeys.size || keys.some((key) => !allowedKeys.has(key))) { + throw new Error('Benchmark summary input has an unsupported shape.'); + } if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark summary unit is invalid.'); } @@ -253,6 +268,7 @@ function validateSummary(value) { unit: value.unit, sourceCommitSha: value.sourceCommitSha, artifactSha256: value.artifactSha256, + ...inputIdentity, documentProfile: value.documentProfile, runtimeId: value.runtimeId, referenceHardwareId: value.referenceHardwareId, @@ -306,6 +322,8 @@ function compare(baseline, current, metric, maxRegressionPercent) { contractVersion: baseline.contractVersion, benchmarkId: baseline.benchmarkId, unit: baseline.unit, + ...(baseline.contractVersion === 3 ? { inputSha256: baseline.inputSha256 } : {}), + ...(baseline.resultingInputSha256 === undefined ? {} : { resultingInputSha256: baseline.resultingInputSha256 }), documentProfile: baseline.documentProfile, runtimeId: baseline.runtimeId, referenceHardwareId: baseline.referenceHardwareId, diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index bf3502a0..4c7dbc06 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -347,6 +347,7 @@ async function createSyntheticRevisionEvidence(args) { 'Autosave benchmark input exceeds the supported size.', ); const revisionModulePath = resolveLocalModule(args.revisionModulePath); + const inputSha256 = createHash('sha256').update(source).digest('hex'); verifyMeasuredModuleDigest( revisionModulePath, args.revisionArtifactSha256, @@ -361,9 +362,10 @@ async function createSyntheticRevisionEvidence(args) { ); } try { - return await revisionModule.createDocumentEnvelopeRevisionEvidenceBytes( + const evidence = await revisionModule.createDocumentEnvelopeRevisionEvidenceBytes( source, ); + return { evidence, inputSha256 }; } catch { throw new Error('Autosave benchmark input must be a valid document envelope.'); } @@ -380,7 +382,7 @@ async function createSyntheticRevisionEvidence(args) { const digestHex = createHash('sha256') .update(JSON.stringify(documentJson)) .digest('hex'); - return Object.freeze({ + const evidence = Object.freeze({ envelope: Object.freeze({ schemaId: 'https://inkspan.io/schemas/document-envelope/v1', schemaVersion: 1, @@ -392,6 +394,10 @@ async function createSyntheticRevisionEvidence(args) { strongEntityTag: `"sha256-${digestHex}"`, }), }); + return { + evidence, + inputSha256: createHash('sha256').update(JSON.stringify(evidence.envelope)).digest('hex'), + }; } async function measureOneEnqueue(createDocumentAutosaveQueue, evidence) { @@ -541,7 +547,7 @@ async function main() { 'Measured autosave module must export createDocumentAutosaveQueue().', ); } - const evidence = await createSyntheticRevisionEvidence(args); + const { evidence, inputSha256 } = await createSyntheticRevisionEvidence(args); const measure = { enqueue: measureOneEnqueue, @@ -571,11 +577,12 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 2, + contractVersion: 3, benchmarkId: `autosave-${args.operation}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, + inputSha256, documentProfile: args.profile, runtimeId: args.runtimeId, referenceHardwareId: args.referenceHardwareId, diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 3bbbb497..9faa380a 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -236,7 +236,10 @@ function readBoundedMarkdown(path) { 'Markdown benchmark input exceeds the supported size.', ); try { - return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + return { + source: new TextDecoder('utf-8', { fatal: true }).decode(bytes), + inputSha256: createHash('sha256').update(bytes).digest('hex'), + }; } catch { throw new Error('Markdown benchmark input must be valid UTF-8.'); } @@ -397,7 +400,7 @@ function writeMeasurementOutput(path, content) { async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); - const source = readBoundedMarkdown(args.inputPath); + const { source, inputSha256 } = readBoundedMarkdown(args.inputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) @@ -460,11 +463,12 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 2, + contractVersion: 3, benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, + inputSha256, documentProfile: args.profile, runtimeId: args.runtimeId, referenceHardwareId: args.referenceHardwareId, diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index b739cd66..67f46d4d 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -420,6 +420,10 @@ async function main() { const source = readBoundedEnvelopeBytes(args.inputPath); const resultingSource = args.resultingInputPath === undefined ? source : readBoundedEnvelopeBytes(args.resultingInputPath); + const inputSha256 = createHash('sha256').update(source).digest('hex'); + const resultingInput = args.operation === 'transition-changed' + ? { resultingInputSha256: createHash('sha256').update(resultingSource).digest('hex') } + : {}; if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) || @@ -489,7 +493,7 @@ async function main() { args.outputPath, `${JSON.stringify( { - contractVersion: 2, + contractVersion: 3, benchmarkId: args.operation === 'canonicalization' ? `envelope-canonicalization-${args.profile}` @@ -497,6 +501,8 @@ async function main() { unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, + inputSha256, + ...resultingInput, documentProfile: args.profile, runtimeId: args.runtimeId, referenceHardwareId: args.referenceHardwareId, diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index ef93d728..ede369c1 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -134,11 +134,8 @@ function validateInput(value) { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Benchmark sample input must be an object.'); } - if (Object.keys(value).some((key) => !BENCHMARK_INPUT_KEYS.has(key))) { - throw new Error('Benchmark sample input contains unsupported fields.'); - } - if (value.contractVersion !== 1 && value.contractVersion !== 2) { - throw new Error('Benchmark sample contractVersion must be 1 or 2.'); + if (![1, 2, 3].includes(value.contractVersion)) { + throw new Error('Benchmark sample contractVersion must be 1, 2 or 3.'); } if ( typeof value.benchmarkId !== 'string' || @@ -146,6 +143,25 @@ function validateInput(value) { ) { throw new Error('Benchmark benchmarkId is invalid.'); } + const inputIdentity = {}; + if (value.contractVersion === 3) { + inputIdentity.inputSha256 = value.inputSha256; + if (value.benchmarkId.startsWith('transition-changed-evidence-')) { + inputIdentity.resultingInputSha256 = value.resultingInputSha256; + if (value.inputSha256 === value.resultingInputSha256) { + throw new Error('Benchmark changed-transition inputs must differ.'); + } + } + for (const digest of Object.values(inputIdentity)) { + if (typeof digest !== 'string' || !SHA256_PATTERN.test(digest)) { + throw new Error('Benchmark input identities must be lowercase SHA-256 digests.'); + } + } + } + const allowedKeys = new Set([...BENCHMARK_INPUT_KEYS, ...Object.keys(inputIdentity)]); + if (Object.keys(value).some((key) => !allowedKeys.has(key))) { + throw new Error('Benchmark sample input contains unsupported fields.'); + } if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark unit is invalid.'); } @@ -207,6 +223,7 @@ function validateInput(value) { unit: value.unit, sourceCommitSha: value.sourceCommitSha, artifactSha256: value.artifactSha256, + ...inputIdentity, documentProfile: value.documentProfile, runtimeId: value.runtimeId, referenceHardwareId: value.referenceHardwareId, @@ -227,6 +244,8 @@ function summarize(input) { unit: input.unit, sourceCommitSha: input.sourceCommitSha, artifactSha256: input.artifactSha256, + ...(input.contractVersion === 3 ? { inputSha256: input.inputSha256 } : {}), + ...(input.resultingInputSha256 === undefined ? {} : { resultingInputSha256: input.resultingInputSha256 }), documentProfile: input.documentProfile, runtimeId: input.runtimeId, referenceHardwareId: input.referenceHardwareId, @@ -247,6 +266,8 @@ function formatSummary(summary) { `unit=${summary.unit}`, `source_commit_sha=${summary.sourceCommitSha}`, `artifact_sha256=${summary.artifactSha256}`, + ...(summary.contractVersion === 3 ? [`input_sha256=${summary.inputSha256}`] : []), + ...(summary.resultingInputSha256 === undefined ? [] : [`resulting_input_sha256=${summary.resultingInputSha256}`]), `document_profile=${summary.documentProfile}`, `runtime_id=${summary.runtimeId}`, `reference_hardware_id=${summary.referenceHardwareId}`, diff --git a/docs/TRD.md b/docs/TRD.md index ee8f0424..4424983c 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -135,7 +135,7 @@ Queued, cancelled, skipped-required, absent, stale-head, predecessor-head, statu Active PR performance research records the first operation invocation without an unrecorded latency warmup, with timing boundaries and generation-comparison limits documented in [`benchmarks/README.md`](../benchmarks/README.md). This harness does not establish buyer-workload latency or a supported document-size guarantee; a measurement-method change starts a new baseline rather than proving a product speedup. -The active research sample/summary contract marks first-invocation JavaScript latency evidence as version 2. Legacy version 1 remains readable, but cross-version comparisons fail closed. Corpus and suite-inventory versions are independent; no editor API or supported-performance promise changes. +The active research sample/summary contract marks input-bound first-invocation JavaScript latency evidence as version 3. It derives SHA-256 identities from the captured bounded input bytes before timing; changed transitions preserve ordered previous/resulting identities. The summarizer preserves those identities and the comparator rejects mismatched inputs even when profile labels match. Legacy versions 1 and 2 remain readable without backfilled identities; cross-version comparisons fail closed. Corpus and suite-inventory versions are independent; no editor API or supported-performance promise changes. See the research harness for the explicit synthetic autosave input representation and claim limits. ## Security, privacy, and operability dependencies diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e51b4751..fc477c7c 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -130,7 +130,7 @@ After P0 blockers and independent non-conflicting lanes permit, prioritize: proposal acceptance without moving identity or persistence into Inkspan; 2. measured large-document latency/memory support envelopes with realistic fixtures and deterministic failure behavior; the active research harness's - [first-invocation accounting and claim limits](../benchmarks/README.md) + [first-invocation accounting, exact input identity and claim limits](../benchmarks/README.md) do not close this buyer-workload gap; 3. CJK IME, touch, and mobile editing assurance with truthful real-device versus emulated support claims; diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index 91afa593..c2509773 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -131,7 +131,7 @@ describe('autosave enqueue performance measurement', () => { samples?: unknown[]; }; expect(evidence).toMatchObject({ - contractVersion: 2, + contractVersion: 3, benchmarkId: 'autosave-enqueue-small', unit: 'ms', sourceCommitSha, diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts index 0961dbec..f803b415 100644 --- a/src/performanceHtmlSerializationMeasurement.test.ts +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -85,10 +85,12 @@ describe('HTML serialization performance measurement', () => { contractVersion: number; benchmarkId: string; unit: string; + inputSha256: string; documentProfile: string; samples: unknown[]; }; - expect(evidence.contractVersion).toBe(2); + expect(evidence.contractVersion).toBe(3); + expect(evidence.inputSha256).toBe(sha256(input)); expect(evidence.benchmarkId).toBe('html-serialization-small'); expect(evidence.unit).toBe('ms'); expect(evidence.documentProfile).toBe('small'); diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index c3c6467a..219414a3 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -14,7 +14,7 @@ import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSamples { - readonly contractVersion: 2; + readonly contractVersion: 3; readonly benchmarkId: string; readonly unit: 'ms'; readonly sourceCommitSha: string; @@ -105,7 +105,7 @@ describe('Markdown runtime measurement contract', () => { readFileSync(samplesPath, 'utf8'), ) as BenchmarkSamples; expect(samples).toMatchObject({ - contractVersion: 2, + contractVersion: 3, benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, @@ -133,7 +133,7 @@ describe('Markdown runtime measurement contract', () => { readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), ) as { sampleCount: number; benchmarkId: string; unit: string }; expect(summary).toMatchObject({ - contractVersion: 2, + contractVersion: 3, sampleCount: 3, benchmarkId: 'markdown-serialization-large', unit: 'ms', diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 786e93f6..33be40b6 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -181,7 +181,8 @@ describe('deterministic benchmark sample statistics', () => { expect(result.status).toBe(1); expect(result.stdout).toBe(''); expect(result.stderr).not.toContain('private-path'); - expect(existsSync(output)).toBe(false); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); } finally { rmSync(root, { recursive: true, force: true }); } diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 864f1333..745bdf79 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -14,7 +14,7 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; interface BenchmarkSamples { - readonly contractVersion: 2; + readonly contractVersion: 3; readonly benchmarkId: string; readonly unit: 'ms'; readonly sourceCommitSha: string; @@ -123,7 +123,7 @@ describe('revision-evidence runtime measurement contract', () => { readFileSync(samplesPath, 'utf8'), ) as BenchmarkSamples; expect(samples).toMatchObject({ - contractVersion: 2, + contractVersion: 3, benchmarkId: 'revision-evidence-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, From b866a1d5815bdbd2d6f3ba4091bd945b30f43cdc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:32:21 +0900 Subject: [PATCH 255/260] test(performance): distinguish captured and prepared input identities Signed-off-by: Seongho Bae --- ...ormanceAutosaveMeasurementContract.test.ts | 6 ++++- ...ormanceMarkdownMeasurementContract.test.ts | 26 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts index c2509773..7e0ac47e 100644 --- a/src/performanceAutosaveMeasurementContract.test.ts +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -173,12 +173,15 @@ describe('autosave enqueue performance measurement', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-coalescing-')); const modulePath = join(directory, 'autosave.mjs'); const outputPath = join(directory, 'samples.json'); - const moduleSource = `let queueCount = 0; + const capturedInput = join(directory, 'captured-input.json'); + const moduleSource = `import { writeFileSync } from 'node:fs'; +let queueCount = 0; export function createDocumentAutosaveQueue(options) { if (++queueCount > 2) throw new Error('Unmeasured invocation'); let active; return { enqueue(evidence) { + writeFileSync(${JSON.stringify(capturedInput)}, JSON.stringify(evidence)); if (active) return active; active = Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); return active; @@ -209,6 +212,7 @@ export function createDocumentAutosaveQueue(options) { expect(result.status).toBe(0); expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toMatchObject({ benchmarkId: 'autosave-coalescing-small', + inputSha256: sha256(readFileSync(capturedInput, 'utf8')), samples: [expect.any(Number), expect.any(Number)], }); } finally { diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 219414a3..b9794b83 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -80,14 +80,14 @@ function measurementArguments( } describe('Markdown runtime measurement contract', () => { - it('writes bounded privacy-safe samples consumable by the canonical summarizer', () => { + it.each(['', '\uFEFF'])('writes bounded privacy-safe samples with UTF-8 prefix %j', (prefix) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-')); const input = join(root, 'large.md'); const modulePath = join(root, 'packed-markdown.mjs'); const samplesPath = join(root, 'samples.json'); const summaryDirectory = join(root, 'summary'); try { - writeFileSync(input, '# Buyer benchmark fixture\n\nSynthetic content only.\n', 'utf8'); + writeFileSync(input, `${prefix}# Buyer benchmark fixture\n\nSynthetic content only.\n`, 'utf8'); writeFileSync( modulePath, "let measuredCalls = 0;\nexport function markdownToHtml(source) { if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); return `

${source.length}

`; }\n", @@ -144,6 +144,28 @@ describe('Markdown runtime measurement contract', () => { } }); + it('identifies captured bytes even if the file changes after reading', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-captured-input-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'serializer.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '\uFEFF# Captured input\n', 'utf8'); + const inputSha256 = fileSha256(input); + writeFileSync(modulePath, `import { writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(input)}, '# Later replacement\\n'); +export function markdownToHtml(source) { + if (source !== '# Captured input\\n') throw new Error('wrong captured input'); + return '

Captured input

'; +}\n`); + execFileSync(process.execPath, measurementArguments(input, modulePath, output), { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }); + expect(fileSha256(input)).not.toBe(inputSha256); + expect(JSON.parse(readFileSync(output, 'utf8'))).toMatchObject({ inputSha256, samples: [expect.any(Number), expect.any(Number), expect.any(Number)] }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it.each([ ['markdown-to-html', 'markdownToHtml', 1], ['markdown-to-html', 'markdownToHtml', 3], From c7d829d91057d2d307133f3038c818ab5f602283 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:33:28 +0900 Subject: [PATCH 256/260] fix(performance): identify the prepared synthetic autosave payload Signed-off-by: Seongho Bae --- benchmarks/README.md | 3 ++- benchmarks/measure-autosave.mjs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 71e8676c..c1ee743b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -28,7 +28,8 @@ use `contractVersion: 3` and additionally identify the captured input bytes with read used by the operation, not a caller-supplied claim or a later file read. Changed transitions also record `resultingInputSha256` in its distinct resulting role. Autosave with no input file hashes the UTF-8 JSON representation of its -fixed synthetic envelope; that input remains synthetic regardless of profile. +fixed prepared revision-evidence payload (envelope and revision), not a file-mode +envelope; that input remains synthetic regardless of profile. Input digests are workload identifiers, not anonymization or authenticity proofs. The summarizer preserves these identities in JSON and text. The comparator diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs index 4c7dbc06..220b37bd 100644 --- a/benchmarks/measure-autosave.mjs +++ b/benchmarks/measure-autosave.mjs @@ -396,7 +396,7 @@ async function createSyntheticRevisionEvidence(args) { }); return { evidence, - inputSha256: createHash('sha256').update(JSON.stringify(evidence.envelope)).digest('hex'), + inputSha256: createHash('sha256').update(JSON.stringify(evidence)).digest('hex'), }; } From 75f195de0f8321848d544c3bb4cd33e0fefb9e8a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:48:30 +0900 Subject: [PATCH 257/260] test(performance): verify packed input identity and document evidence limits Signed-off-by: Seongho Bae --- benchmarks/README.md | 3 + docs/doctoring/performance-input-identity.md | 88 +++++++++++++++++++ ...ormancePackedArtifactSuiteContract.test.ts | 10 +++ 3 files changed, 101 insertions(+) create mode 100644 docs/doctoring/performance-input-identity.md diff --git a/benchmarks/README.md b/benchmarks/README.md index c1ee743b..6670cb40 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,6 +7,9 @@ buyer-workload performance, real-device input behavior, or the 20 ms target. Envelope fixtures contain plain paragraphs; Markdown list, table, and image syntax inside those paragraphs is not a rich editor document tree. +The [captured-input identity research record](../docs/doctoring/performance-input-identity.md) +documents the failure, rejected alternatives and exact-source regression evidence. + ## First-invocation accounting The revision, Markdown/HTML, and autosave latency producers record the first diff --git a/docs/doctoring/performance-input-identity.md b/docs/doctoring/performance-input-identity.md new file mode 100644 index 00000000..21049963 --- /dev/null +++ b/docs/doctoring/performance-input-identity.md @@ -0,0 +1,88 @@ +# Captured-input identity for performance evidence + +Status: Research only; active PR #379, not protected-main implementation +Date: 2026-09-06 +Owner: [performance support-envelope issue #375](https://github.com/ContextualWisdomLab/inkspan/issues/375) + +## Problem and decision + +A document profile alone does not identify the workload. The direct latency +commands accepted different input files under the same profile while their +receipts identified only the source, artifact, runtime and reference hardware. +Two such summaries could therefore appear comparable despite measuring different +documents. The locked synthetic suite already checked its fixtures; this defect +was at the direct producer/summary/comparison boundary. + +Version 3 derives an input SHA-256 from the already bounded read before calling +the measured operation. Changed transitions retain ordered previous and resulting +input identities. The summarizer preserves them and the comparator rejects a +mismatch before computing a regression or improvement. SHA-256 identifies bytes; +it does not establish workload realism, confidentiality, authenticity or runtime +correctness (National Institute of Standards and Technology [NIST], 2015). + +The implementation reuses the existing Node crypto module. Its hash API accepts +byte buffers directly, avoiding a decode/re-encode step that could discard a +UTF-8 byte-order mark (Node.js, n.d.). Hashing is outside the existing timer; +first-invocation accounting, sample counts and scenario assertions are unchanged. + +## Alternatives and failure prevention + +- Caller-supplied digests were rejected because they repeat the unverified claim. +- Hashing a second file read was rejected because the file can change after the + operation's input is captured. A regression test replaces it during module + loading and requires the original captured identity. +- Adding optional fields to version 2 was rejected because old readers could + compare evidence without enforcing input identity. Versions 1 and 2 keep their + original shapes and meanings; cross-version comparisons remain prohibited. +- No-file autosave hashes the actual prepared synthetic revision-evidence payload + (envelope plus revision). Hashing just its envelope would confuse this distinct + preparation with file-mode envelope input. A fixture captures the value passed + to the queue and verifies its identity. + +Malformed, missing, extra, legacy-backfilled and identical changed-transition +identities fail closed. No document contents or paths are added to receipts. +Digests are not anonymization: private or guessable inputs must not be published +merely because their content was replaced by a hash. + +## Evidence lineage + +Local retained evidence directory: +`/private/tmp/inkspan-input-identity-evidence.3O7Jac`. + +| Source revision | Experiment | Result and scope | +| --- | --- | --- | +| `69bdcc5c4d8ba757fb84d6e96c77fb2704cecf7c` | Three producer contracts | 6 failed identity checks, 29 passed; `producers-identity-red-69bd.log` | +| `ce48badc22ecf45cec8ff3f14e40a323deaae0fc` | Five-file chain RED | 27 failed, 58 passed; 7 failures were incorrect new directory-absence assertions, subsequently corrected to require no evidence files; `chain-identity-red.log` | +| `c4fba276b1ea8725a61f354bc0c3e26c58aa6c54` | Six focused files | 86 passed; `chain-identity-green.log` | +| `b866a1d5815bdbd2d6f3ba4091bd945b30f43cdc` | Captured/prepared input distinction | 1 synthetic-payload identity failure, 13 passed; `captured-input-red.log` | +| `c7d829d91057d2d307133f3038c818ab5f602283` | Same captured/prepared checks | 14 passed; `captured-input-green.log`; TypeScript check also passed | +| `c7d829d91057d2d307133f3038c818ab5f602283` | All 41 performance files, two workers | 158 passed, 8 failed, 1 worker RPC error; `all-performance-c7d829.log`, 325.70 seconds. Failures include unchanged test/subprocess deadlines and three null child exit statuses; this is not complete acceptance | + +These are harness correctness experiments, not latency improvements. Broader +acceptance must use the final exact head and retain every failed denominator. +Historical samples are neither rewritten nor backfilled. The immutable source +commits preserve the changes even if the local temporary evidence expires. + +Run the directly affected checks from the repository root: + +```sh +pnpm exec vitest run src/performanceMarkdownMeasurementContract.test.ts src/performanceRevisionMeasurementContract.test.ts src/performanceAutosaveMeasurementContract.test.ts src/performanceHtmlSerializationMeasurement.test.ts src/performanceMeasurementStatisticsContract.test.ts src/performanceRegressionComparatorContract.test.ts --maxWorkers=2 +``` + +## Remaining work and authority + +A real-workload corpus, fresh same-generation baselines, profiling and verified +optimizations remain necessary. This change does not prove the 20 ms target, +supported document sizes, browser interaction latency or release acceptance. +The corpus lock and suite inventory keep their independent version 1 contracts. +No editor API, host authority or accepted ADR changes. See the +[harness contract](../../benchmarks/README.md), [TRD](../TRD.md) and +[buyer gap](../product-technical-gap-baseline.md). + +## References + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). https://doi.org/10.6028/NIST.FIPS.180-4 + +Node.js. (n.d.). *Crypto: Hash update and digest*. Node.js v24 documentation. +Retrieved September 6, 2026, from https://nodejs.org/docs/latest-v24.x/api/crypto.html#hashupdatedata-inputencoding diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index e891b0bf..b2d9ffa5 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -207,6 +207,7 @@ describe('packed artifact benchmark suite contract', () => { }, ); + expect(result.error).toBeUndefined(); expect(result.status).toBe(0); expect(result.stderr).toBe(''); expect(JSON.parse(result.stdout.trim())).toMatchObject({ @@ -250,6 +251,10 @@ describe('packed artifact benchmark suite contract', () => { ), ) as { benchmarkId?: unknown; samples?: unknown[] }; expect(autosaveSamples.benchmarkId).toBe('autosave-enqueue-small'); + expect(autosaveSamples).toMatchObject({ + contractVersion: 3, + inputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.envelope.json'))), + }); expect(autosaveSamples.samples).toHaveLength(2); const coalescingSamples = JSON.parse( @@ -291,6 +296,11 @@ describe('packed artifact benchmark suite contract', () => { join(directory, 'evidence', 'transition-changed', 'samples.json'), 'utf8', )); expect(changedSamples.benchmarkId).toBe('transition-changed-evidence-small'); + expect(changedSamples).toMatchObject({ + contractVersion: 3, + inputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.envelope.json'))), + resultingInputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.changed.envelope.json'))), + }); expect(changedSamples.samples).toHaveLength(2); } else { expect(JSON.parse(result.stdout)).not.toHaveProperty('changedTransitionSamples'); From 98f068a8f667447f947660e18283e8c5c7b236c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:59:46 +0900 Subject: [PATCH 258/260] test(perf): reject inconsistent serialization samples Signed-off-by: Seongho Bae --- ...ormanceMarkdownMeasurementContract.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index b9794b83..44f1dff1 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -196,6 +196,32 @@ export function markdownToHtml(source) { } }); + it.each([ + ['markdown-to-html', 'markdownToHtml', 2], + ['markdown-to-html', 'markdownToHtml', 3], + ['html-to-markdown', 'htmlToMarkdown', 2], + ['html-to-markdown', 'htmlToMarkdown', 3], + ] as const)('rejects changing %s output on invocation %i', (operation, exportName, changedCall) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-serialization-changing-result-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'serializer.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n'); + writeFileSync(modulePath, `let calls = 0; +export function ${exportName}() { return ++calls === ${changedCall} ? 'private changed output' : ''; }\n`); + const args = measurementArguments(input, modulePath, output); + args.splice(5, 0, '--operation', operation); + const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Measured serialization output changed for identical input.'); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed before output when the measured module lacks the public serializer', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); const input = join(root, 'small.md'); From ea2e9982a067b2652dc413314854e4fdc9de30f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:00:18 +0900 Subject: [PATCH 259/260] fix(perf): reject changing serialization results Signed-off-by: Seongho Bae --- benchmarks/README.md | 10 ++++++++++ benchmarks/measure-markdown.mjs | 5 +++++ src/performanceMarkdownMeasurementContract.test.ts | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 6670cb40..2b946d06 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -19,6 +19,16 @@ samples can be published. Module loading and input preparation remain outside the timer; autosave queue setup and coalescing-scenario preparation retain their existing timer boundaries. This is not process-startup latency. +Markdown/HTML producers also require every measured output to equal the first +measured output for the same captured input. Empty strings are valid reference +outputs. A changed result rejects the entire acquisition without publishing +samples or output content; there is no extra warmup call. Equality is checked +after the timer and retains one output in memory. Start a fresh baseline after +this validation change: process state can differ even though the operation's +timer boundaries and sample schema remain unchanged. Equal outputs alone do +not prove conversion fidelity; workload-specific correctness checks remain +required, and a one-sample run cannot establish repeatability. + Samples recorded before this change retain their original meaning. Start a new baseline for the new measurement method; a difference between those generations is not evidence of a product speedup. Keep each generation's raw diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 9faa380a..2265b862 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -429,6 +429,7 @@ async function main() { } const samples = []; + let firstOutput; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); const output = runMeasuredSerialization( @@ -440,6 +441,10 @@ async function main() { if (typeof output !== 'string') { throw new Error(contract.returnFailure); } + if (index === 0) firstOutput = output; + else if (output !== firstOutput) { + throw new Error('Measured serialization output changed for identical input.'); + } if (!Number.isFinite(elapsed) || elapsed < 0) { throw new Error('Markdown measurement produced invalid runtime evidence.'); } diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 44f1dff1..38949a04 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -201,7 +201,7 @@ export function markdownToHtml(source) { ['markdown-to-html', 'markdownToHtml', 3], ['html-to-markdown', 'htmlToMarkdown', 2], ['html-to-markdown', 'htmlToMarkdown', 3], - ] as const)('rejects changing %s output on invocation %i', (operation, exportName, changedCall) => { + ] as const)('rejects changing %s output from %s on invocation %i', (operation, exportName, changedCall) => { const root = mkdtempSync(join(tmpdir(), 'inkspan-serialization-changing-result-')); const input = join(root, 'input.md'); const modulePath = join(root, 'serializer.mjs'); From 7fb2068698bddd38264af169b4897f0eb4ee8802 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:01:10 +0900 Subject: [PATCH 260/260] docs(perf): distinguish stable output from conversion fidelity Signed-off-by: Seongho Bae --- docs/TRD.md | 7 +++++++ docs/product-technical-gap-baseline.md | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/TRD.md b/docs/TRD.md index 4424983c..f8d2d123 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -135,6 +135,13 @@ Queued, cancelled, skipped-required, absent, stale-head, predecessor-head, statu Active PR performance research records the first operation invocation without an unrecorded latency warmup, with timing boundaries and generation-comparison limits documented in [`benchmarks/README.md`](../benchmarks/README.md). This harness does not establish buyer-workload latency or a supported document-size guarantee; a measurement-method change starts a new baseline rather than proving a product speedup. +The active Markdown/HTML research producer rejects an acquisition when repeated +serialization of the same captured input returns different strings. Comparison +occurs after timing and retains the first measured output; no output content is +published in the failure. This consistency check cannot establish conversion +fidelity or repeatability from one sample. It starts a fresh measurement baseline +without changing the sample schema or published editor contract. + The active research sample/summary contract marks input-bound first-invocation JavaScript latency evidence as version 3. It derives SHA-256 identities from the captured bounded input bytes before timing; changed transitions preserve ordered previous/resulting identities. The summarizer preserves those identities and the comparator rejects mismatched inputs even when profile labels match. Legacy versions 1 and 2 remain readable without backfilled identities; cross-version comparisons fail closed. Corpus and suite-inventory versions are independent; no editor API or supported-performance promise changes. See the research harness for the explicit synthetic autosave input representation and claim limits. ## Security, privacy, and operability dependencies diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index fc477c7c..67578a8b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -131,7 +131,9 @@ After P0 blockers and independent non-conflicting lanes permit, prioritize: 2. measured large-document latency/memory support envelopes with realistic fixtures and deterministic failure behavior; the active research harness's [first-invocation accounting, exact input identity and claim limits](../benchmarks/README.md) - do not close this buyer-workload gap; + do not close this buyer-workload gap. Reject inconsistent repeated conversion + results before publishing latency samples; consistency alone does not replace + workload-specific fidelity checks or realistic document evidence; 3. CJK IME, touch, and mobile editing assurance with truthful real-device versus emulated support claims; 4. an executable packed-package reference host proving SSR/hydration, native