diff --git a/.github/workflows/performance-evidence.yml b/.github/workflows/performance-evidence.yml new file mode 100644 index 000000000..a2bc00a82 --- /dev/null +++ b/.github/workflows/performance-evidence.yml @@ -0,0 +1,106 @@ +name: Performance Evidence + +on: + pull_request: + paths: + - '.github/workflows/performance-evidence.yml' + - 'benchmarks/**' + - 'src/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig.json' + - 'vite.config.ts' + schedule: + - cron: '17 3 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + packed-artifact-smoke: + name: packed-artifact-${{ matrix.profile }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: ${{ fromJSON(github.event_name == 'pull_request' && '{"include":[{"profile":"small","samples":3}]}' || '{"include":[{"profile":"small","samples":25},{"profile":"medium","samples":25},{"profile":"large","samples":25},{"profile":"stress","samples":25}]}') }} + steps: + - name: Checkout exact pull-request head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Verify exact checkout + shell: bash + env: + INKSPAN_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + actual_head="$(git rev-parse HEAD)" + test "$actual_head" = "$INKSPAN_EXPECTED_HEAD_SHA" + + - name: Set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: pnpm + + - name: Install immutable dependencies + run: pnpm install --frozen-lockfile + + - name: Build packed library + run: pnpm build + + - name: Produce bounded packed-artifact smoke evidence + shell: bash + run: | + set -euo pipefail + corpus_dir="${RUNNER_TEMP}/inkspan-benchmark-corpus" + package_dir="${RUNNER_TEMP}/inkspan-package" + evidence_dir="${RUNNER_TEMP}/inkspan-performance-evidence" + + rm -rf "$corpus_dir" "$package_dir" "$evidence_dir" + node benchmarks/generate-corpus.mjs --output "$corpus_dir" + + mkdir -p "$package_dir" + pnpm pack --pack-destination "$package_dir" + mapfile -t packages < <(find "$package_dir" -maxdepth 1 -type f -name '*.tgz' -print) + test "${#packages[@]}" -eq 1 + package="${packages[0]}" + package_sha256="$(sha256sum "$package" | cut -d' ' -f1)" + source_sha="$(git rev-parse HEAD)" + runtime_id="node-$(node -p 'process.versions.node')" + + node benchmarks/run-current-suite.mjs \ + --input "$corpus_dir/${{ matrix.profile }}.md" \ + --html-input "$corpus_dir/${{ matrix.profile }}.html" \ + --revision-input "$corpus_dir/${{ matrix.profile }}.envelope.json" \ + --package-tarball "$package" \ + --package-sha256 "$package_sha256" \ + --profile "${{ matrix.profile }}" \ + --samples "${{ matrix.samples }}" \ + --source-commit-sha "$source_sha" \ + --runtime-id "$runtime_id" \ + --reference-hardware-id github-actions-ubuntu-24.04-x64 \ + --resulting-input "$corpus_dir/${{ matrix.profile }}.changed.envelope.json" \ + --output "$evidence_dir" + + - name: Upload bounded performance evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0.0 + with: + name: performance-${{ matrix.profile }}-${{ github.event.pull_request.head.sha || github.sha }} + path: ${{ runner.temp }}/inkspan-performance-evidence + if-no-files-found: error + retention-days: 5 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..2b946d064 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,86 @@ +# Performance evidence development harness + +This harness is research infrastructure, not a published latency or supported +document-size guarantee. The generated corpus is synthetic and suitable for +repeatable regression probes and harness contract tests. It does not establish +buyer-workload performance, real-device input behavior, or the 20 ms target. +Envelope fixtures contain plain paragraphs; Markdown list, table, and image +syntax inside those paragraphs is not a rich editor document tree. + +The [captured-input identity research record](../docs/doctoring/performance-input-identity.md) +documents the failure, rejected alternatives and exact-source regression evidence. + +## First-invocation accounting + +The revision, Markdown/HTML, and autosave latency producers record the first +operation invocation and every subsequent requested sample, without an +unrecorded warmup invocation. Each result must pass its scenario checks before +samples can be published. Module loading and input preparation remain outside +the timer; autosave queue setup and coalescing-scenario preparation retain +their existing timer boundaries. This is not process-startup latency. + +Markdown/HTML producers also require every measured output to equal the first +measured output for the same captured input. Empty strings are valid reference +outputs. A changed result rejects the entire acquisition without publishing +samples or output content; there is no extra warmup call. Equality is checked +after the timer and retains one output in memory. Start a fresh baseline after +this validation change: process state can differ even though the operation's +timer boundaries and sample schema remain unchanged. Equal outputs alone do +not prove conversion fidelity; workload-specific correctness checks remain +required, and a one-sample run cannot establish repeatability. + +Samples recorded before this change retain their original meaning. Start a +new baseline for the new measurement method; a difference between those +generations is not evidence of a product speedup. Keep each generation's raw +samples and exact source revision. Office rendering and the separate memory +settling analysis are unchanged. + +Version 2 introduced this first-invocation method. New JavaScript latency samples +use `contractVersion: 3` and additionally identify the captured input bytes with +`inputSha256`. The digest is derived before measurement from the same bounded +read used by the operation, not a caller-supplied claim or a later file read. +Changed transitions also record `resultingInputSha256` in its distinct resulting +role. Autosave with no input file hashes the UTF-8 JSON representation of its +fixed prepared revision-evidence payload (envelope and revision), not a file-mode +envelope; that input remains synthetic regardless of profile. +Input digests are workload identifiers, not anonymization or authenticity proofs. + +The summarizer preserves these identities in JSON and text. The comparator +rejects different versions or input identities before calculating a verdict, +even when document profiles match. Versions 1 and 2 remain readable with their +original meanings; Office output remains version 1. Do not backfill identities, +relabel historical samples, or present a cross-generation difference as a +speedup. Update producers and evidence readers together. The suite inventory and +corpus locks retain their independent version 1 contracts; published editor APIs +and timer boundaries are unchanged. This does not establish a real-world corpus. + +## Transition scenarios + +| Operation | Inputs | Required result | Metric prefix | +| --- | --- | --- | --- | +| `transition` | Same captured envelope twice | Unchanged, equal revision digests | `transition-evidence` | +| `transition-changed` | Explicit previous and resulting envelopes | Changed, unequal revision digests | `transition-changed-evidence` | + +The profile suffix remains `small`, `medium`, `large`, or `stress`. Existing +transition samples retain their no-op meaning. Never compare the two scenarios +as an optimization result; the comparison tool rejects mismatched metric IDs. + +For `measure-revision-evidence.mjs`, append +`--operation transition-changed --resulting-input ` +immediately before `--output`. Other operations reject `--resulting-input`. +Both inputs use the same bounded, non-symlink file reader. An output cannot +overwrite either input, including through a hard link. Invalid results publish +no sample file, and sample output contains neither document content nor paths. + +The packed `run-current-suite.mjs` accepts `--resulting-input` immediately before +`--output`, with or without `--html-input`. It verifies the resulting fixture +against `changedEnvelopeBytes` and `changedEnvelopeSha256` in `corpus.lock.json`. +The generator adds one plain paragraph to produce each +`.changed.envelope.json`; all earlier fixture bytes and digests remain +unchanged. Omitting the new flag preserves the earlier suite. The +`performance-evidence.yml` workflow exercises both transition scenarios. + +Commit a clean source checkout and rebuild the package before recording evidence. +Keep raw samples with the exact source SHA, packed-artifact digest, Node runtime, +hardware identity, and sample count. A new scenario starts a separate baseline; +synthetic smoke evidence cannot close the realistic-workload support-envelope gap. diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs new file mode 100644 index 000000000..f15d3d830 --- /dev/null +++ b/benchmarks/analyze-memory-settling.mjs @@ -0,0 +1,309 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PREFIX = 'editor-lifecycle-retained-memory-'; +const BENCHMARK_ID_PATTERN = + /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const EVIDENCE_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'warmupSamples', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 6 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--window-size' || + argv[3].trim().length === 0 || + argv[4] !== '--max-growth-bytes' || + argv[5].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/analyze-memory-settling.mjs --input --window-size --max-growth-bytes ', + ); + } + + const windowSize = Number(argv[3]); + if (!Number.isSafeInteger(windowSize) || windowSize <= 0) { + throw new Error('Memory settling window size must be a positive safe integer.'); + } + + const maxGrowthBytes = Number(argv[5]); + if (!Number.isFinite(maxGrowthBytes) || maxGrowthBytes < 0) { + throw new Error( + 'Memory settling max growth bytes must be a finite non-negative number.', + ); + } + + return Object.freeze({ + inputPath: resolve(argv[1]), + windowSize, + maxGrowthBytes, + }); +} + +function inspectEvidenceInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Memory settling evidence input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectEvidenceInputPath(path); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Memory settling evidence input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Memory settling evidence input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error( + 'Memory settling evidence input exceeds the supported size.', + ); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Memory settling evidence input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Memory settling evidence input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateEvidence(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Memory settling evidence input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== EVIDENCE_KEYS.size || + keys.some((key) => !EVIDENCE_KEYS.has(key)) + ) { + throw new Error('Memory settling evidence input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Memory settling evidence contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Memory settling evidence benchmarkId is invalid.'); + } + if (value.unit !== 'bytes') { + throw new Error('Memory settling evidence unit must be bytes.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Memory settling evidence sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Memory settling evidence artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Memory settling evidence documentProfile is invalid.'); + } + if ( + value.benchmarkId.slice(BENCHMARK_ID_PREFIX.length) !== value.documentProfile + ) { + throw new Error( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Memory settling evidence runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Memory settling evidence referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.warmupSamples) || + value.warmupSamples < 0 || + value.warmupSamples > MAX_SAMPLES + ) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES || + value.samples.some( + (sample) => + !Number.isSafeInteger(sample) || sample < 0, + ) + ) { + throw new Error( + 'Memory settling evidence samples must be bounded non-negative safe integers.', + ); + } + if (value.warmupSamples >= value.samples.length) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + warmupSamples: value.warmupSamples, + samples: Object.freeze([...value.samples]), + }); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[midpoint]; + + const left = sorted[midpoint - 1]; + const right = sorted[midpoint]; + const distance = right - left; + const wholeMidpoint = left + Math.floor(distance / 2); + if (distance % 2 === 1 && wholeMidpoint >= 2 ** 52) { + throw new Error( + 'Memory settling window median must be exactly representable.', + ); + } + return wholeMidpoint + (distance % 2) / 2; +} + +function analyze(evidence, windowSize, maxGrowthBytes) { + const settledSamples = evidence.samples.slice(evidence.warmupSamples); + if (settledSamples.length < windowSize * 2) { + throw new Error( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } + + const firstWindowMedianBytes = median(settledSamples.slice(0, windowSize)); + const lastWindowMedianBytes = median(settledSamples.slice(-windowSize)); + const retainedGrowthBytes = lastWindowMedianBytes - firstWindowMedianBytes; + + return Object.freeze({ + contractVersion: 1, + benchmarkId: evidence.benchmarkId, + unit: evidence.unit, + sourceCommitSha: evidence.sourceCommitSha, + artifactSha256: evidence.artifactSha256, + documentProfile: evidence.documentProfile, + runtimeId: evidence.runtimeId, + referenceHardwareId: evidence.referenceHardwareId, + sampleCount: evidence.samples.length, + warmupSamples: evidence.warmupSamples, + windowSize, + firstWindowMedianBytes, + lastWindowMedianBytes, + retainedGrowthBytes, + maxGrowthBytes, + passed: retainedGrowthBytes <= maxGrowthBytes, + }); +} + +function main() { + const { inputPath, windowSize, maxGrowthBytes } = resolveArguments( + process.argv.slice(2), + ); + const evidence = validateEvidence(readBoundedJson(inputPath)); + const result = analyze(evidence, windowSize, maxGrowthBytes); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Memory settling analysis failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs new file mode 100644 index 000000000..eda882a76 --- /dev/null +++ b/benchmarks/compare-summaries.mjs @@ -0,0 +1,362 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|transition-changed-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; +const UNITS = new Set(['ms', 'bytes']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); +const SUMMARY_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', + 'minimum', + 'p50', + 'p75', + 'p95', + 'maximum', +]); +const COMPARABLE_FIELDS = [ + 'contractVersion', + 'benchmarkId', + 'unit', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', + 'inputSha256', + 'resultingInputSha256', +]; +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 8 || + argv[0] !== '--baseline' || + argv[1].length === 0 || + argv[2] !== '--current' || + argv[3].length === 0 || + argv[4] !== '--metric' || + !METRICS.has(argv[5]) || + argv[6] !== '--max-regression-percent' || + argv[7].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/compare-summaries.mjs --baseline --current --metric --max-regression-percent ', + ); + } + + const maxRegressionPercent = Number(argv[7]); + if (!Number.isFinite(maxRegressionPercent) || maxRegressionPercent < 0) { + throw new Error( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } + + return Object.freeze({ + baselinePath: resolve(argv[1]), + currentPath: resolve(argv[3]), + metric: argv[5], + maxRegressionPercent, + }); +} + +function inspectSummaryInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectSummaryInputPath(path); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Benchmark summary input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Benchmark summary input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateSummary(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark summary input must be an object.'); + } + if (![1, 2, 3].includes(value.contractVersion)) { + throw new Error('Benchmark summary contractVersion must be 1, 2 or 3.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark summary benchmarkId is invalid.'); + } + const inputIdentity = {}; + if (value.contractVersion === 3) { + inputIdentity.inputSha256 = value.inputSha256; + if (value.benchmarkId.startsWith('transition-changed-evidence-')) { + inputIdentity.resultingInputSha256 = value.resultingInputSha256; + if (value.inputSha256 === value.resultingInputSha256) { + throw new Error('Benchmark summary changed-transition inputs must differ.'); + } + } + for (const digest of Object.values(inputIdentity)) { + if (typeof digest !== 'string' || !SHA256_PATTERN.test(digest)) { + throw new Error('Benchmark summary input identities must be lowercase SHA-256 digests.'); + } + } + } + const keys = Object.keys(value); + const allowedKeys = new Set([...SUMMARY_KEYS, ...Object.keys(inputIdentity)]); + if (keys.length !== allowedKeys.size || keys.some((key) => !allowedKeys.has(key))) { + throw new Error('Benchmark summary input has an unsupported shape.'); + } + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { + throw new Error('Benchmark summary unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Benchmark summary sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Benchmark summary artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark summary documentProfile is invalid.'); + } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark summary profile must match documentProfile.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark summary runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark summary referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.sampleCount) || + value.sampleCount <= 0 || + value.sampleCount > MAX_SAMPLES + ) { + throw new Error('Benchmark summary sampleCount is invalid.'); + } + if (value.percentileMethod !== 'nearest-rank') { + throw new Error('Benchmark summary percentileMethod is invalid.'); + } + + const measurements = [ + value.minimum, + value.p50, + value.p75, + value.p95, + value.maximum, + ]; + if ( + measurements.some( + (measurement) => + typeof measurement !== 'number' || + !Number.isFinite(measurement) || + measurement < 0, + ) + ) { + throw new Error( + 'Benchmark summary measurements must be finite non-negative numbers.', + ); + } + for (let index = 1; index < measurements.length; index += 1) { + if (measurements[index] < measurements[index - 1]) { + throw new Error('Benchmark summary percentile ordering is invalid.'); + } + } + + return Object.freeze({ + contractVersion: value.contractVersion, + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + ...inputIdentity, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + sampleCount: value.sampleCount, + percentileMethod: value.percentileMethod, + minimum: value.minimum, + p50: value.p50, + p75: value.p75, + p95: value.p95, + maximum: value.maximum, + }); +} + +function assertComparable(baseline, current) { + for (const field of COMPARABLE_FIELDS) { + if (baseline[field] !== current[field]) { + throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); + } + } + if (baseline.artifactSha256 === current.artifactSha256) { + throw new Error( + 'Benchmark summaries must identify distinct measured artifacts.', + ); + } +} + +function normalizePercent(value) { + const rounded = Number(value.toFixed(12)); + return Object.is(rounded, -0) ? 0 : rounded; +} + +function compare(baseline, current, metric, maxRegressionPercent) { + assertComparable(baseline, current); + const baselineValue = baseline[metric]; + const currentValue = current[metric]; + if (baselineValue === 0 && currentValue !== 0) { + throw new Error( + 'Benchmark regression percent is undefined for a zero non-matching baseline.', + ); + } + const regressionPercent = + baselineValue === 0 + ? 0 + : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + if (!Number.isFinite(regressionPercent)) { + throw new Error( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } + return Object.freeze({ + contractVersion: baseline.contractVersion, + benchmarkId: baseline.benchmarkId, + unit: baseline.unit, + ...(baseline.contractVersion === 3 ? { inputSha256: baseline.inputSha256 } : {}), + ...(baseline.resultingInputSha256 === undefined ? {} : { resultingInputSha256: baseline.resultingInputSha256 }), + documentProfile: baseline.documentProfile, + runtimeId: baseline.runtimeId, + referenceHardwareId: baseline.referenceHardwareId, + sampleCount: baseline.sampleCount, + percentileMethod: baseline.percentileMethod, + metric, + baselineSourceCommitSha: baseline.sourceCommitSha, + baselineArtifactSha256: baseline.artifactSha256, + currentSourceCommitSha: current.sourceCommitSha, + currentArtifactSha256: current.artifactSha256, + baselineValue, + currentValue, + maxRegressionPercent, + regressionPercent, + passed: regressionPercent <= maxRegressionPercent, + }); +} + +function main() { + const { baselinePath, currentPath, metric, maxRegressionPercent } = + resolveArguments(process.argv.slice(2)); + const baseline = validateSummary(readBoundedJson(baselinePath)); + const current = validateSummary(readBoundedJson(currentPath)); + const result = compare(baseline, current, metric, maxRegressionPercent); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark comparison failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json new file mode 100644 index 000000000..2ec368071 --- /dev/null +++ b/benchmarks/corpus.lock.json @@ -0,0 +1,58 @@ +{ + "contractVersion": 1, + "synthetic": true, + "scripts": [ + "English", + "Korean", + "Japanese", + "Chinese", + "Vietnamese", + "mixed" + ], + "profiles": { + "small": { + "sections": 1, + "bytes": 2152, + "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea", + "htmlBytes": 2225, + "htmlSha256": "f06ba85c8fa00c819dcb992232fbad2205915a3066abc29e275c590f32fb5356", + "envelopeBytes": 4236, + "envelopeSha256": "7ee1425e1f57808a3a4d11987a0a6bc2aec97d311457f5456e82f1adbc48c810", + "changedEnvelopeBytes": 4319, + "changedEnvelopeSha256": "aafa73ad4ed397de59a55fd21cd8c021dfee9eb2a828e732dc693d040c13ba64" + }, + "medium": { + "sections": 8, + "bytes": 15712, + "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c", + "htmlBytes": 15807, + "htmlSha256": "09dcb4384632d6b5525707d21c22788e838f7c68ae5e55e020ff0367e342bc5a", + "envelopeBytes": 30018, + "envelopeSha256": "47fa0cd11ffec5f9c5513a935180d1d7249157c46a0e115d1de938be86fa4459", + "changedEnvelopeBytes": 30101, + "changedEnvelopeSha256": "68cc2e64e3129cfdd49b9fd16f2bea134f6f7275a8e671873f5913d4adde12ab" + }, + "large": { + "sections": 32, + "bytes": 62199, + "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392", + "htmlBytes": 62365, + "htmlSha256": "3952dd5228a1e0f34bc294019e317c5ae38a26d4b4606cca8344e4dc36221102", + "envelopeBytes": 118409, + "envelopeSha256": "c93143659a352cffaa1ff817b0efa0ba4b63f8410bbb7d923799b827a36aa655", + "changedEnvelopeBytes": 118492, + "changedEnvelopeSha256": "d95afab352b1813395f7e95bd8c2c8c983583ca1577e1ae56fee60e961516bbf" + }, + "stress": { + "sections": 128, + "bytes": 248152, + "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928", + "htmlBytes": 248607, + "htmlSha256": "1d5d42bd92b7121a4ca59a542edf881d0edb438d7de68c09d3c075703bf3e4cb", + "envelopeBytes": 471978, + "envelopeSha256": "ebc47c89671e39af349a4cb65ba15ed3138a3eee187245e520abe80d877bcd6c", + "changedEnvelopeBytes": 472061, + "changedEnvelopeSha256": "4a160cde5bf9c7ac9aa8aeeae0bce13d10632bd055944dc7c61a70403c55746a" + } + } +} diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs new file mode 100644 index 000000000..74971e6e6 --- /dev/null +++ b/benchmarks/generate-corpus.mjs @@ -0,0 +1,218 @@ +import { createHash } from 'node:crypto'; +import { + lstatSync, + mkdirSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +const RASTER_FIXTURES = Object.freeze([ + Object.freeze({ + dimensions: '1x1', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNQcEj4DwADBAHAX3ZiygAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '16x16', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR42mNQcEj4TwlmGDVg1IBRA4aLAQDSpr8QG8NsyQAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '64x64', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAZklEQVR42u3QQREAAAQAMFFEEUX/EuRw9liBRVbPZyFAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgTct/Bk8ZbOy3oMAAAAAElFTkSuQmCC', + }), +]); +const SCRIPT_PARAGRAPHS = [ + 'English: Deterministic authoring keeps document state explicit and reviewable.', + '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', + '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', + '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', + 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', +]; +const PROFILE_SECTIONS = Object.freeze({ + small: 1, + medium: 8, + large: 32, + stress: 128, +}); +const SCRIPT_LABELS = Object.freeze([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', +]); + +let outputWriteCounter = 0; + +function assertSafeDirectoryChain(directoryPath) { + let currentPath = directoryPath; + while (true) { + const existing = lstatSync(currentPath, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isDirectory()) { + throw new Error( + 'Benchmark corpus output directory must not traverse symbolic links or non-directories.', + ); + } + const parentPath = dirname(currentPath); + if (parentPath === currentPath) { + break; + } + currentPath = parentPath; + } +} + +function writeRegularOutput(path, content) { + const existing = lstatSync(path, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isFile()) { + throw new Error('Benchmark corpus output must be a regular file.'); + } + + const temporaryPath = `${path}.tmp-${process.pid}-${outputWriteCounter}`; + outputWriteCounter += 1; + try { + writeFileSync(temporaryPath, content, { flag: 'wx' }); + renameSync(temporaryPath, path); + } finally { + rmSync(temporaryPath, { force: true }); + } +} + +function buildSection(index) { + const id = String(index).padStart(4, '0'); + const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { + const row = String(rowIndex + 1).padStart(2, '0'); + return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; + }); + const rasterRows = RASTER_FIXTURES.map( + ({ dimensions, base64 }) => + `![synthetic raster ${dimensions} ${id}](data:image/png;base64,${base64})`, + ); + return [ + `# Synthetic section ${id}`, + '', + ...SCRIPT_PARAGRAPHS, + '', + `## Nested list ${id}`, + `- item ${id}-a`, + ` - item ${id}-a-1`, + ` - item ${id}-a-2`, + `- item ${id}-b`, + '', + `> Synthetic blockquote ${id}: benchmark text only; no production content.`, + '', + '```text', + `fixture=${id}; authority=none; network=none`, + '```', + '', + `[synthetic safe link ${id}](https://example.invalid/inkspan/${id})`, + '', + '| c01 | c02 | c03 | c04 | c05 | c06 |', + '| --- | --- | --- | --- | --- | --- |', + ...tableRows, + '', + ...rasterRows, + '', + '---', + '', + ].join('\n'); +} + +function buildProfile(profile, sectionCount) { + return [ + `# Inkspan deterministic benchmark fixture: ${profile}`, + '', + 'Synthetic fixture only. No customer, tenant, prompt, credential, or model data.', + 'Scripts: English, Korean, Japanese, Chinese, Vietnamese, and mixed-script structure.', + '', + ...Array.from({ length: sectionCount }, (_, index) => buildSection(index + 1)), + ].join('\n'); +} + +function buildEnvelope(body) { + const content = body.split('\n').map((line) => + line.length === 0 + ? { type: 'paragraph' } + : { type: 'paragraph', content: [{ type: 'text', text: line }] }, + ); + return `${JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content, + }, + })}\n`; +} + +function buildHtml(profile, body) { + const escapedBody = body + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>'); + return `
${escapedBody}
\n`; +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error('Usage: node benchmarks/generate-corpus.mjs --output '); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +assertSafeDirectoryChain(outputDirectory); +mkdirSync(outputDirectory, { recursive: true }); +assertSafeDirectoryChain(outputDirectory); + +const profileManifest = {}; +for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { + const body = buildProfile(profile, sections); + const bytes = Buffer.from(body, 'utf8'); + const htmlBytes = Buffer.from(buildHtml(profile, body), 'utf8'); + const envelopeBytes = Buffer.from(buildEnvelope(body), 'utf8'); + const changedEnvelopeBytes = Buffer.from( + buildEnvelope(`${body}\nSynthetic appended edit.`), 'utf8', + ); + writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); + writeRegularOutput(resolve(outputDirectory, `${profile}.html`), htmlBytes); + writeRegularOutput( + resolve(outputDirectory, `${profile}.envelope.json`), + envelopeBytes, + ); + writeRegularOutput( + resolve(outputDirectory, `${profile}.changed.envelope.json`), + changedEnvelopeBytes, + ); + profileManifest[profile] = Object.freeze({ + sections, + bytes: bytes.byteLength, + sha256: sha256(bytes), + htmlBytes: htmlBytes.byteLength, + htmlSha256: sha256(htmlBytes), + envelopeBytes: envelopeBytes.byteLength, + envelopeSha256: sha256(envelopeBytes), + changedEnvelopeBytes: changedEnvelopeBytes.byteLength, + changedEnvelopeSha256: sha256(changedEnvelopeBytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + scripts: SCRIPT_LABELS, + profiles: profileManifest, +}); +writeRegularOutput( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, +); diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs new file mode 100644 index 000000000..e3264cc4d --- /dev/null +++ b/benchmarks/generate-office-fixtures.mjs @@ -0,0 +1,302 @@ +import { createHash } from 'node:crypto'; +import { + closeSync, + constants, + fstatSync, + ftruncateSync, + lstatSync, + mkdirSync, + openSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +const DOCX_PROFILE_PAGES = Object.freeze({ + small: 2, + page120: 120, +}); + +const PPTX_PROFILE_SLIDES = Object.freeze({ + small: 2, + slide120: 120, +}); + +const EXCEL_MAX_COLUMNS = 16_384; +const MULTILINGUAL_PARAGRAPH = + 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; +const WRITE_NOFOLLOW = + constants.O_WRONLY | + constants.O_CREAT | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const OUTPUT_DIRECTORY_ERROR = + 'Office fixture output directory must be a non-symlink directory.'; + +function buildDocxPage(pageNumber) { + const page = String(pageNumber).padStart(3, '0'); + return [ + Object.freeze({ + type: 'heading', + level: 1, + text: `Synthetic page ${page}`, + }), + Object.freeze({ + type: 'paragraph', + text: `${MULTILINGUAL_PARAGRAPH} Page ${page}.`, + alignment: 'justify', + }), + Object.freeze({ + type: 'rich_paragraph', + runs: Object.freeze([ + Object.freeze({ text: `Page ${page} summary: `, bold: true }), + Object.freeze({ text: 'deterministic ', italic: true }), + Object.freeze({ text: 'Office rendering fixture.', underline: true }), + ]), + }), + Object.freeze({ + type: 'bullet_list', + ordered: false, + items: Object.freeze([ + `page ${page} item A`, + `page ${page} item B`, + `page ${page} item C`, + ]), + }), + Object.freeze({ + type: 'table', + headers: Object.freeze(['Page', 'Metric', 'Value']), + rows: Object.freeze([ + Object.freeze([page, 'latency-sample', pageNumber]), + Object.freeze([page, 'memory-sample', pageNumber * 2]), + Object.freeze([page, 'revision-sample', pageNumber * 3]), + Object.freeze([page, 'render-sample', pageNumber * 4]), + ]), + }), + ]; +} + +function buildDocxRequest(profile, pages) { + const blocks = []; + for (let page = 1; page <= pages; page += 1) { + blocks.push(...buildDocxPage(page)); + if (page < pages) { + blocks.push(Object.freeze({ type: 'page_break' })); + } + } + return Object.freeze({ + format: 'docx', + title: `Inkspan synthetic DOCX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + subject: 'Deterministic synthetic performance fixture', + blocks: Object.freeze(blocks), + }); +} + +function buildXlsxRequest(profile) { + if (profile === 'wide16384') { + const row = Array.from( + { length: EXCEL_MAX_COLUMNS }, + (_, index) => `C${String(index + 1).padStart(5, '0')}`, + ); + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: wide16384', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Wide16384', + rows: Object.freeze([Object.freeze(row)]), + freeze_panes: 'XFD1048576', + }), + ]), + }); + } + + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: small', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Synthetic', + header_row: true, + auto_filter: true, + freeze_panes: 'B2', + rows: Object.freeze([ + Object.freeze(['Language', 'Text', 'Latency', 'Memory']), + Object.freeze(['한국어', '합성 성능 문서', 1, 2]), + Object.freeze(['日本語', '合成性能文書', 3, 4]), + Object.freeze(['中文 / Tiếng Việt', '合成文档 / tài liệu tổng hợp', 5, 6]), + ]), + }), + ]), + }); +} + +function buildPptxSlide(slideNumber) { + const slide = String(slideNumber).padStart(3, '0'); + return Object.freeze({ + title: `한국어 합성 슬라이드 ${slide}`, + bullets: Object.freeze([ + `English deterministic slide ${slide}`, + Object.freeze({ text: `日本語 合成スライド ${slide}`, level: 0 }), + Object.freeze({ text: `中文 合成幻灯片 ${slide}`, level: 1 }), + Object.freeze({ text: `Tiếng Việt trang chiếu ${slide}`, level: 1 }), + ]), + }); +} + +function buildPptxRequest(profile, slides) { + return Object.freeze({ + format: 'pptx', + title: `Inkspan synthetic PPTX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + slides: Object.freeze( + Array.from({ length: slides }, (_, index) => buildPptxSlide(index + 1)), + ), + }); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error( + 'Usage: node benchmarks/generate-office-fixtures.mjs --output ', + ); + } + return resolve(argv[1]); +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputDirectoryAncestors(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function writeOutputFile(outputPath, bytes) { + let pathMetadata; + try { + pathMetadata = lstatSync(outputPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output path could not be inspected.'); + } + if (pathMetadata?.isSymbolicLink()) { + throw new Error('Office fixture output must be a regular non-symlink file.'); + } + if (pathMetadata !== undefined && !pathMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + + let descriptor; + try { + descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); + const descriptorMetadata = fstatSync(descriptor); + if (!descriptorMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + if (descriptorMetadata.nlink !== 1) { + throw new Error('Office fixture output must not be multiply linked.'); + } + ftruncateSync(descriptor, 0); + writeFileSync(descriptor, bytes); + } catch (error) { + if ( + error instanceof Error && + (error.message === 'Office fixture output must be a regular file.' || + error.message === 'Office fixture output must be a regular non-symlink file.' || + error.message === 'Office fixture output must not be multiply linked.') + ) { + throw error; + } + throw new Error('Office fixture output could not be written safely.'); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function writeFixture(outputDirectory, fileName, request, units) { + const body = `${JSON.stringify(request, null, 2)}\n`; + const bytes = Buffer.from(body, 'utf8'); + writeOutputFile(resolve(outputDirectory, fileName), bytes); + return Object.freeze({ + units, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); +try { + mkdirSync(outputDirectory, { recursive: true }); +} catch { + throw new Error('Office fixture output directory could not be prepared.'); +} +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); + +const docx = {}; +for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { + docx[profile] = writeFixture( + outputDirectory, + `docx-${profile}.json`, + buildDocxRequest(profile, pages), + pages, + ); +} + +const xlsx = Object.freeze({ + small: writeFixture( + outputDirectory, + 'xlsx-small.json', + buildXlsxRequest('small'), + 4, + ), + wide16384: writeFixture( + outputDirectory, + 'xlsx-wide16384.json', + buildXlsxRequest('wide16384'), + EXCEL_MAX_COLUMNS, + ), +}); + +const pptx = {}; +for (const [profile, slides] of Object.entries(PPTX_PROFILE_SLIDES)) { + pptx[profile] = writeFixture( + outputDirectory, + `pptx-${profile}.json`, + buildPptxRequest(profile, slides), + slides, + ); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + formats: Object.freeze({ + docx: Object.freeze(docx), + xlsx, + pptx: Object.freeze(pptx), + }), +}); +writeOutputFile( + resolve(outputDirectory, 'manifest.json'), + Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), +); \ No newline at end of file diff --git a/benchmarks/measure-autosave.mjs b/benchmarks/measure-autosave.mjs new file mode 100644 index 000000000..220b37bd5 --- /dev/null +++ b/benchmarks/measure-autosave.mjs @@ -0,0 +1,606 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Autosave benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = 'Autosave benchmark output must not already exist.'; +const SYNTHETIC_DOCUMENT_LABEL = 'Synthetic autosave benchmark document'; + +function resolveArguments(argv) { + const legacyFlags = [ + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + const inputFlags = [ + '--input', + '--revision-module', + '--revision-artifact-sha256', + ...legacyFlags, + ]; + const operationFlags = [...legacyFlags.slice(0, -1), '--operation', '--output']; + const inputOperationFlags = [ + ...inputFlags.slice(0, -1), + '--operation', + '--output', + ]; + const expectedFlags = + argv.length === inputOperationFlags.length * 2 + ? inputOperationFlags + : argv.length === inputFlags.length * 2 + ? inputFlags + : argv.length === operationFlags.length * 2 + ? operationFlags + : legacyFlags; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-autosave.mjs [--input --revision-module --revision-artifact-sha256 ] --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Autosave benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error( + 'Autosave benchmark sample count must be an integer from 1 to 1000.', + ); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Autosave benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Autosave benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const revisionArtifactSha256 = values['--revision-artifact-sha256']; + if ( + values['--input'] !== undefined && + !SHA256_PATTERN.test(revisionArtifactSha256) + ) { + throw new Error( + 'Autosave benchmark revision artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Autosave benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Autosave benchmark reference hardware ID is invalid.'); + } + const operation = values['--operation'] ?? 'enqueue'; + if ( + operation !== 'enqueue' && + operation !== 'coalescing' && + operation !== 'commit' + ) { + throw new Error('Autosave benchmark operation is invalid.'); + } + + return Object.freeze({ + inputPath: + values['--input'] === undefined ? null : resolve(values['--input']), + revisionModulePath: values['--revision-module'], + revisionArtifactSha256, + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + operation, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidFileMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured autosave module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured autosave module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured autosave module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured autosave module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured autosave module must be a local regular file.', + 'Measured autosave module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Autosave benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Autosave benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Autosave benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) throw new Error(OUTPUT_DIRECTORY_ERROR); + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Autosave benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured autosave module could not be loaded.'); + } +} + +async function createSyntheticRevisionEvidence(args) { + const inputPath = args.inputPath; + if (inputPath !== null) { + const source = readBoundedRegularFile( + inputPath, + MAX_INPUT_BYTES, + 'Autosave benchmark input must be a regular non-symlink file.', + 'Autosave benchmark input exceeds the supported size.', + ); + const revisionModulePath = resolveLocalModule(args.revisionModulePath); + const inputSha256 = createHash('sha256').update(source).digest('hex'); + verifyMeasuredModuleDigest( + revisionModulePath, + args.revisionArtifactSha256, + ); + const revisionModule = await loadMeasuredModule(revisionModulePath); + if ( + typeof revisionModule.createDocumentEnvelopeRevisionEvidenceBytes !== + 'function' + ) { + throw new Error( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + } + try { + const evidence = await revisionModule.createDocumentEnvelopeRevisionEvidenceBytes( + source, + ); + return { evidence, inputSha256 }; + } catch { + throw new Error('Autosave benchmark input must be a valid document envelope.'); + } + } + const textNode = Object.freeze({ type: 'text', text: SYNTHETIC_DOCUMENT_LABEL }); + const paragraph = Object.freeze({ + type: 'paragraph', + content: Object.freeze([textNode]), + }); + const documentJson = Object.freeze({ + type: 'doc', + content: Object.freeze([paragraph]), + }); + const digestHex = createHash('sha256') + .update(JSON.stringify(documentJson)) + .digest('hex'); + const evidence = Object.freeze({ + envelope: Object.freeze({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson, + }), + revision: Object.freeze({ + algorithm: 'SHA-256', + digestHex, + strongEntityTag: `"sha256-${digestHex}"`, + }), + }); + return { + evidence, + inputSha256: createHash('sha256').update(JSON.stringify(evidence)).digest('hex'), + }; +} + +async function measureOneEnqueue(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return Object.freeze({ status: 'saved' }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + + const start = performance.now(); + const outcome = await queue.enqueue(evidence); + const elapsed = performance.now() - start; + if ( + typeof outcome !== 'object' || + outcome === null || + outcome.status !== 'saved' || + saveCalls !== 1 + ) { + throw new Error('Measured autosave enqueue result is invalid.'); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Autosave measurement produced invalid runtime evidence.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + +async function measureOneCoalescing(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + let releaseSave; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return new Promise((resolve) => { + releaseSave = () => resolve(Object.freeze({ status: 'saved' })); + }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + const active = queue.enqueue(evidence); + await Promise.resolve(); + if (typeof releaseSave !== 'function') { + throw new Error('Measured autosave coalescing setup is invalid.'); + } + + const start = performance.now(); + const coalesced = queue.enqueue(evidence); + const elapsed = performance.now() - start; + if (coalesced !== active || saveCalls !== 1) { + throw new Error('Measured autosave coalescing result is invalid.'); + } + releaseSave(); + const outcome = await active; + if (outcome?.status !== 'saved') { + throw new Error('Measured autosave coalescing result is invalid.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + +async function measureOneCommit(createDocumentAutosaveQueue, evidence) { + let saveCalls = 0; + let releaseSave; + const queue = createDocumentAutosaveQueue({ + save: () => { + saveCalls += 1; + return new Promise((resolve) => { + releaseSave = () => resolve(Object.freeze({ status: 'saved' })); + }); + }, + }); + if ( + typeof queue !== 'object' || + queue === null || + typeof queue.enqueue !== 'function' + ) { + throw new Error('Measured autosave module returned an invalid queue.'); + } + const active = queue.enqueue(evidence); + await Promise.resolve(); + if (typeof releaseSave !== 'function' || saveCalls !== 1) { + throw new Error('Measured autosave commit setup is invalid.'); + } + + const start = performance.now(); + releaseSave(); + const outcome = await active; + const elapsed = performance.now() - start; + if (outcome?.status !== 'saved') { + throw new Error('Measured autosave commit result is invalid.'); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Autosave measurement produced invalid runtime evidence.'); + } + if (typeof queue.close === 'function') await queue.close(); + return elapsed; +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Autosave benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Autosave benchmark output could not be written.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Autosave benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + if (typeof measuredModule.createDocumentAutosaveQueue !== 'function') { + throw new Error( + 'Measured autosave module must export createDocumentAutosaveQueue().', + ); + } + const { evidence, inputSha256 } = await createSyntheticRevisionEvidence(args); + + const measure = { + enqueue: measureOneEnqueue, + coalescing: measureOneCoalescing, + commit: measureOneCommit, + }[args.operation]; + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + samples.push( + await measure(measuredModule.createDocumentAutosaveQueue, evidence), + ); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Autosave benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Autosave benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) throw new Error(OUTPUT_EXISTS_ERROR); + + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 3, + benchmarkId: `autosave-${args.operation}-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + inputSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Autosave benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs new file mode 100644 index 000000000..2265b862d --- /dev/null +++ b/benchmarks/measure-markdown.mjs @@ -0,0 +1,497 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SERIALIZATION_OPERATIONS = new Set([ + 'markdown-to-html', + 'html-to-markdown', +]); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Markdown benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Markdown benchmark output must not already exist.'; +const LEGACY_FLAGS = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const OPERATION_FLAGS = Object.freeze([ + '--input', + '--module', + '--operation', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function resolveArguments(argv) { + let values; + let operation = 'markdown-to-html'; + if (matchesArguments(argv, OPERATION_FLAGS)) { + values = valuesForArguments(argv, OPERATION_FLAGS); + operation = values['--operation']; + if (!SERIALIZATION_OPERATIONS.has(operation)) { + throw new Error('Markdown benchmark serialization operation is invalid.'); + } + } else if (matchesArguments(argv, LEGACY_FLAGS)) { + values = valuesForArguments(argv, LEGACY_FLAGS); + } else { + throw new Error( + 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Markdown benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Markdown benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Markdown benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Markdown benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Markdown benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Markdown benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + operation, + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedMarkdown(path) { + const bytes = readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Markdown benchmark input must be a regular non-symlink file.', + 'Markdown benchmark input exceeds the supported size.', + ); + try { + return { + source: new TextDecoder('utf-8', { fatal: true }).decode(bytes), + inputSha256: createHash('sha256').update(bytes).digest('hex'), + }; + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured Markdown module must be a local regular file.', + 'Measured Markdown module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured Markdown module could not be loaded.'); + } +} + +function serializationContract(operation) { + if (operation === 'html-to-markdown') { + return Object.freeze({ + exportName: 'htmlToMarkdown', + benchmarkPrefix: 'html-serialization', + executionFailure: 'Measured htmlToMarkdown() execution failed.', + returnFailure: 'Measured htmlToMarkdown() must return a string.', + }); + } + return Object.freeze({ + exportName: 'markdownToHtml', + benchmarkPrefix: 'markdown-serialization', + executionFailure: 'Measured markdownToHtml() execution failed.', + returnFailure: 'Measured markdownToHtml() must return a string.', + }); +} + +function runMeasuredSerialization(serializer, source, failureMessage) { + try { + return serializer(source); + } catch { + throw new Error(failureMessage); + } +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const { source, inputSha256 } = readBoundedMarkdown(args.inputPath); + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { + throw new Error('Markdown benchmark output must not overwrite its input.'); + } + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Markdown benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + const contract = serializationContract(args.operation); + const serializer = measuredModule[contract.exportName]; + if (typeof serializer !== 'function') { + throw new Error( + `Measured Markdown module must export ${contract.exportName}().`, + ); + } + + const samples = []; + let firstOutput; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + const output = runMeasuredSerialization( + serializer, + source, + contract.executionFailure, + ); + const elapsed = performance.now() - start; + if (typeof output !== 'string') { + throw new Error(contract.returnFailure); + } + if (index === 0) firstOutput = output; + else if (output !== firstOutput) { + throw new Error('Measured serialization output changed for identical input.'); + } + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Markdown measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Markdown benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 3, + benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + inputSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Markdown benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs new file mode 100644 index 000000000..67f46d4d8 --- /dev/null +++ b/benchmarks/measure-revision-evidence.mjs @@ -0,0 +1,526 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Revision benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Revision benchmark output must not already exist.'; + +function resolveArguments(argv) { + const baseFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + const operationFlags = [...baseFlags.slice(0, -1), '--operation', '--output']; + const changedTransitionFlags = [ + ...operationFlags.slice(0, -1), '--resulting-input', '--output', + ]; + const expectedFlags = + [baseFlags, operationFlags, changedTransitionFlags].find( + (flags) => argv.length === flags.length * 2, + ) ?? baseFlags; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Revision benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Revision benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Revision benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Revision benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Revision benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Revision benchmark reference hardware ID is invalid.'); + } + const operation = values['--operation'] ?? 'revision'; + if ( + operation !== 'revision' && + operation !== 'transition' && + operation !== 'transition-changed' && + operation !== 'canonicalization' + ) { + throw new Error('Revision benchmark operation is invalid.'); + } + if ((operation === 'transition-changed') !== (values['--resulting-input'] !== undefined)) { + throw new Error('Only changed-transition measurement requires a resulting input.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + resultingInputPath: values['--resulting-input'] === undefined + ? undefined : resolve(values['--resulting-input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + operation, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedEnvelopeBytes(path) { + return readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Revision benchmark input must be a regular non-symlink file.', + 'Revision benchmark input exceeds the supported size.', + ); +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured revision module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured revision module must be a local regular file.', + 'Measured revision module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Revision benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured revision module could not be loaded.'); + } +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } +} + +async function runMeasuredEvidence(createEvidence, source, resultingSource, operation) { + const isTransition = operation === 'transition' || operation === 'transition-changed'; + let digestCalls = 0; + const canonicalizationDigestProvider = { + digest(algorithm, canonicalSource) { + digestCalls += 1; + if ( + algorithm !== 'SHA-256' || + !ArrayBuffer.isView(canonicalSource) || + canonicalSource.byteLength === 0 + ) { + throw new Error('invalid canonical source'); + } + return Promise.resolve(new Uint8Array(32).buffer); + }, + }; + let evidence; + try { + evidence = + isTransition + ? await createEvidence(source, resultingSource) + : await createEvidence( + source, + undefined, + operation === 'canonicalization' + ? canonicalizationDigestProvider + : undefined, + ); + } catch { + throw new Error('Measured revision-evidence execution failed.'); + } + + let revisions; + try { + if (typeof evidence !== 'object' || evidence === null) { + throw new Error('invalid revision evidence'); + } + revisions = + isTransition + ? [evidence.previousRevision, evidence.resultingRevision] + : [evidence.revision]; + if ( + (isTransition && evidence.changed !== (operation === 'transition-changed')) || + (operation === 'canonicalization' && digestCalls !== 1) || + revisions.some( + (revision) => typeof revision !== 'object' || revision === null, + ) || + revisions.some( + (revision) => + typeof revision.digestHex !== 'string' || + !SHA256_PATTERN.test(revision.digestHex), + ) + ) { + throw new Error('invalid revision evidence'); + } + if (isTransition && (revisions[0].digestHex !== revisions[1].digestHex) !== evidence.changed) { + throw new Error('invalid transition revision pair'); + } + } catch { + throw new Error('Measured revision-evidence result is invalid.'); + } + +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedEnvelopeBytes(args.inputPath); + const resultingSource = args.resultingInputPath === undefined + ? source : readBoundedEnvelopeBytes(args.resultingInputPath); + const inputSha256 = createHash('sha256').update(source).digest('hex'); + const resultingInput = args.operation === 'transition-changed' + ? { resultingInputSha256: createHash('sha256').update(resultingSource).digest('hex') } + : {}; + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) || + (args.resultingInputPath !== undefined && ( + args.resultingInputPath === args.outputPath || + refersToSameFile(args.resultingInputPath, args.outputPath) + )) + ) { + throw new Error('Revision benchmark output must not overwrite its input.'); + } + if (args.operation === 'transition-changed' && source.equals(resultingSource)) { + throw new Error('Changed-transition benchmark inputs must differ.'); + } + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Revision benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + const isTransition = args.operation === 'transition' || args.operation === 'transition-changed'; + const createEvidence = + isTransition + ? measuredModule.createDocumentEnvelopeTransitionEvidenceBytes + : measuredModule.createDocumentEnvelopeRevisionEvidenceBytes; + if (typeof createEvidence !== 'function') { + throw new Error( + `Measured revision module must export ${ + isTransition + ? 'createDocumentEnvelopeTransitionEvidenceBytes' + : 'createDocumentEnvelopeRevisionEvidenceBytes' + }().`, + ); + } + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + await runMeasuredEvidence(createEvidence, source, resultingSource, args.operation); + const elapsed = performance.now() - start; + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Revision measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Revision benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 3, + benchmarkId: + args.operation === 'canonicalization' + ? `envelope-canonicalization-${args.profile}` + : `${args.operation}-evidence-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + inputSha256, + ...resultingInput, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Revision benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json new file mode 100644 index 000000000..01f9920e8 --- /dev/null +++ b/benchmarks/office-fixtures.lock.json @@ -0,0 +1,42 @@ +{ + "contractVersion": 1, + "synthetic": true, + "formats": { + "docx": { + "small": { + "units": 2, + "bytes": 2972, + "sha256": "c356496b106f5348e98b00d3c1e18185165653a2c2bda083acd7456c96b2eab3" + }, + "page120": { + "units": 120, + "bytes": 169737, + "sha256": "e5d90408c6061051ab1674d931b99d478fadb00a7ecbcb7025b4fa478cfbb507" + } + }, + "xlsx": { + "small": { + "units": 4, + "bytes": 723, + "sha256": "a267a6208de0e24559c200d047404d14e2263581e0a3af845f355817536883bf" + }, + "wide16384": { + "units": 16384, + "bytes": 327941, + "sha256": "0afdb216fda720cd506d7c24284c2740f326167d4859dc1a7b845c4a91b972ec" + } + }, + "pptx": { + "small": { + "units": 2, + "bytes": 970, + "sha256": "6c1a8ae1d2307a278ac97903eb3160a6a0de57b695b2b8772844af2c8cfa2ce1" + }, + "slide120": { + "units": 120, + "bytes": 50061, + "sha256": "f5994ff30752581fd3b3e5210ff59b281692ea268ce4f0b540e278d134dbd6ee" + } + } + } +} diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs new file mode 100644 index 000000000..6d0474ef1 --- /dev/null +++ b/benchmarks/run-current-suite-core.mjs @@ -0,0 +1,817 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedChangedFlags = Object.freeze([ + ...packedFlags.slice(0, -1), '--resulting-input', '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; +const AUTOSAVE_MODULE_ENTRY = 'package/dist/cwl-autosave.js'; +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; +const OUTPUT_DIRECTORY_EXISTS_ERROR = + 'Benchmark suite output directory must not already exist.'; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function sharedArguments(values) { + return Object.freeze({ + documentProfile: values['--profile'], + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], + resultingInputPath: values['--resulting-input'], + outputDirectory: resolve(values['--output']), + }); +} + +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function autosaveMeasurementArguments({ + modulePath, + artifactSha256, + revisionModulePath, + revisionArtifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + shared.revisionInputPath, + '--revision-module', + revisionModulePath, + '--revision-artifact-sha256', + revisionArtifactSha256, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + +function resolveArguments(argv) { + const matchedPackedFlags = [packedFlags, packedChangedFlags].find( + (flags) => matchesArguments(argv, flags), + ); + if (matchedPackedFlags !== undefined) { + const values = valuesForArguments(argv, matchedPackedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (!existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + throw new Error(OUTPUT_DIRECTORY_EXISTS_ERROR); + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectory(path); + if (created === undefined || !created.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + AUTOSAVE_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const autosaveModuleBytes = readPackedEntry( + args.packageTarballPath, + AUTOSAVE_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + const autosaveModulePath = join(temporaryDirectory, 'cwl-autosave.mjs'); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + writeFileSync(autosaveModulePath, autosaveModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + autosaveModulePath, + autosaveArtifactSha256: moduleSha256(autosaveModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function runMeasurementAndSummary({ + measurementScript, + measurementArguments: argumentsList, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { + runBoundedNodeScript( + measurementScript, + [...argumentsList, '--output', samplesPath], + measurementFailure, + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + summaryFailure, + ); +} + +function runSuite(args, markdownArguments, revisionArguments, autosaveArguments) { + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + const transitionSamplesPath = resolve( + args.outputDirectory, + 'transition', + 'samples.json', + ); + const transitionSummaryDirectory = resolve( + args.outputDirectory, + 'transition', + 'summary', + ); + const canonicalizationSamplesPath = resolve( + args.outputDirectory, + 'envelope-canonicalization', + 'samples.json', + ); + const canonicalizationSummaryDirectory = resolve( + args.outputDirectory, + 'envelope-canonicalization', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [...revisionArguments, '--operation', 'transition'], + samplesPath: transitionSamplesPath, + summaryDirectory: transitionSummaryDirectory, + measurementFailure: 'Benchmark suite transition measurement failed.', + summaryFailure: 'Benchmark suite transition summary failed.', + }); + if (autosaveArguments !== null) { + if (args.resultingInputPath !== undefined) { + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [ + ...revisionArguments, '--operation', 'transition-changed', + '--resulting-input', args.resultingInputPath, + ], + samplesPath: resolve(args.outputDirectory, 'transition-changed', 'samples.json'), + summaryDirectory: resolve(args.outputDirectory, 'transition-changed', 'summary'), + measurementFailure: 'Benchmark suite changed-transition measurement failed.', + summaryFailure: 'Benchmark suite changed-transition summary failed.', + }); + } + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: [ + ...revisionArguments, + '--operation', + 'canonicalization', + ], + samplesPath: canonicalizationSamplesPath, + summaryDirectory: canonicalizationSummaryDirectory, + measurementFailure: 'Benchmark suite envelope canonicalization measurement failed.', + summaryFailure: 'Benchmark suite envelope canonicalization summary failed.', + }); + const autosaveSamplesPath = resolve( + args.outputDirectory, + 'autosave', + 'samples.json', + ); + const autosaveSummaryDirectory = resolve( + args.outputDirectory, + 'autosave', + 'summary', + ); + const autosaveCoalescingSamplesPath = resolve( + args.outputDirectory, + 'autosave-coalescing', + 'samples.json', + ); + const autosaveCoalescingSummaryDirectory = resolve( + args.outputDirectory, + 'autosave-coalescing', + 'summary', + ); + const autosaveCommitSamplesPath = resolve( + args.outputDirectory, + 'autosave-commit', + 'samples.json', + ); + const autosaveCommitSummaryDirectory = resolve( + args.outputDirectory, + 'autosave-commit', + 'summary', + ); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: autosaveArguments, + samplesPath: autosaveSamplesPath, + summaryDirectory: autosaveSummaryDirectory, + measurementFailure: 'Benchmark suite autosave measurement failed.', + summaryFailure: 'Benchmark suite autosave summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: [...autosaveArguments, '--operation', 'coalescing'], + samplesPath: autosaveCoalescingSamplesPath, + summaryDirectory: autosaveCoalescingSummaryDirectory, + measurementFailure: 'Benchmark suite autosave coalescing measurement failed.', + summaryFailure: 'Benchmark suite autosave coalescing summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-autosave.mjs', + measurementArguments: [...autosaveArguments, '--operation', 'commit'], + samplesPath: autosaveCommitSamplesPath, + summaryDirectory: autosaveCommitSummaryDirectory, + measurementFailure: 'Benchmark suite autosave commit measurement failed.', + summaryFailure: 'Benchmark suite autosave commit summary failed.', + }); + } +} + +function suiteManifest(args, packageEvidence, includeAutosave) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + transitionSamples: 'transition/samples.json', + transitionSummaryJson: 'transition/summary/summary.json', + transitionSummaryText: 'transition/summary/summary.txt', + ...(args.resultingInputPath === undefined ? {} : { + changedTransitionSamples: 'transition-changed/samples.json', + changedTransitionSummaryJson: 'transition-changed/summary/summary.json', + changedTransitionSummaryText: 'transition-changed/summary/summary.txt', + }), + ...(includeAutosave + ? { + canonicalizationSamples: 'envelope-canonicalization/samples.json', + canonicalizationSummaryJson: + 'envelope-canonicalization/summary/summary.json', + canonicalizationSummaryText: + 'envelope-canonicalization/summary/summary.txt', + autosaveSamples: 'autosave/samples.json', + autosaveSummaryJson: 'autosave/summary/summary.json', + autosaveSummaryText: 'autosave/summary/summary.txt', + autosaveCoalescingSamples: 'autosave-coalescing/samples.json', + autosaveCoalescingSummaryJson: + 'autosave-coalescing/summary/summary.json', + autosaveCoalescingSummaryText: + 'autosave-coalescing/summary/summary.txt', + autosaveCommitSamples: 'autosave-commit/samples.json', + autosaveCommitSummaryJson: 'autosave-commit/summary/summary.json', + autosaveCommitSummaryText: 'autosave-commit/summary/summary.txt', + } + : {}), + status: 'completed', + }); +} + +function main(argv) { + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const autosaveArguments = + resolved.mode === 'packed' + ? autosaveMeasurementArguments({ + modulePath: preparedPackage.autosaveModulePath, + artifactSha256: preparedPackage.autosaveArtifactSha256, + revisionModulePath: preparedPackage.revisionModulePath, + revisionArtifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : null; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; + try { + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments, autosaveArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(shared.outputDirectory); + } + throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } + + process.stdout.write( + `${JSON.stringify( + suiteManifest(shared, packageEvidence, resolved.mode === 'packed'), + )}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs new file mode 100644 index 000000000..f0869af9e --- /dev/null +++ b/benchmarks/run-current-suite.mjs @@ -0,0 +1,633 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const markdownMeasurementPath = resolve( + benchmarkDirectory, + 'measure-markdown.mjs', +); +const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const PACKED_MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const CORPUS_LOCK_PATH = resolve(benchmarkDirectory, 'corpus.lock.json'); +const MAX_CORPUS_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_CORPUS_LOCK_BYTES = 1024 * 1024; +const CORPUS_MISMATCH_ERROR = + 'Benchmark suite inputs must match the committed corpus profile.'; +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const htmlLegacyFlags = Object.freeze([ + '--input', + '--html-input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedHtmlFlags = Object.freeze([ + '--input', + '--html-input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedChangedFlags = Object.freeze([ + ...packedFlags.slice(0, -1), '--resulting-input', '--output', +]); +const packedHtmlChangedFlags = Object.freeze([ + ...packedHtmlFlags.slice(0, -1), '--resulting-input', '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function argumentsForFlags(values, flags) { + return flags.flatMap((flag) => [flag, values[flag]]); +} + +function matchingFlags(argv) { + if (matchesArguments(argv, packedChangedFlags)) return packedChangedFlags; + if (matchesArguments(argv, packedHtmlChangedFlags)) return packedHtmlChangedFlags; + if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; + if (matchesArguments(argv, packedHtmlFlags)) return packedHtmlFlags; + if (matchesArguments(argv, packedFlags)) return packedFlags; + if (matchesArguments(argv, legacyFlags)) return legacyFlags; + return null; +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertFreshOutputDirectory(argv) { + const flags = matchingFlags(argv); + if (flags === null) return; + const outputValueIndex = flags.indexOf('--output') * 2 + 1; + const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); + + let current = outputDirectory; + while (true) { + const metadata = inspectOutputPath(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + if (current === outputDirectory && metadata !== undefined) { + if (!metadata.isDirectory()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + throw new Error('Benchmark suite output directory must not already exist.'); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function claimedLegacySourceCommitSha(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) { + return valuesForArguments(argv, htmlLegacyFlags)['--source-commit-sha']; + } + if (matchesArguments(argv, legacyFlags)) { + return valuesForArguments(argv, legacyFlags)['--source-commit-sha']; + } + return undefined; +} + +function readRegularFileSnapshot(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readPackedTarballSnapshot(path) { + return readRegularFileSnapshot( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function assertPackedCorpusInputs(argv) { + const flags = matchingFlags(argv); + if (!flags?.includes('--package-tarball')) return; + + const values = valuesForArguments(argv, flags); + let lock; + try { + lock = JSON.parse( + readRegularFileSnapshot( + CORPUS_LOCK_PATH, + MAX_CORPUS_LOCK_BYTES, + CORPUS_MISMATCH_ERROR, + CORPUS_MISMATCH_ERROR, + ).toString('utf8'), + ); + } catch { + throw new Error(CORPUS_MISMATCH_ERROR); + } + const profile = lock?.profiles?.[values['--profile']]; + const inputs = [ + ['--input', 'bytes', 'sha256'], + ['--revision-input', 'envelopeBytes', 'envelopeSha256'], + ...(flags.includes('--html-input') + ? [['--html-input', 'htmlBytes', 'htmlSha256']] + : []), + ...(flags.includes('--resulting-input') + ? [['--resulting-input', 'changedEnvelopeBytes', 'changedEnvelopeSha256']] + : []), + ]; + try { + for (const [flag, byteKey, digestKey] of inputs) { + const bytes = readRegularFileSnapshot( + values[flag], + MAX_CORPUS_INPUT_BYTES, + CORPUS_MISMATCH_ERROR, + CORPUS_MISMATCH_ERROR, + ); + if ( + bytes.byteLength !== profile?.[byteKey] || + createHash('sha256').update(bytes).digest('hex') !== profile?.[digestKey] + ) { + throw new Error(CORPUS_MISMATCH_ERROR); + } + } + } catch { + throw new Error(CORPUS_MISMATCH_ERROR); + } +} + +function snapshotPackedArguments(argv) { + const flags = matchingFlags(argv); + if (!flags?.includes('--package-tarball')) { + return Object.freeze({ argv, temporaryDirectory: null }); + } + + const packageTarballValueIndex = + flags.indexOf('--package-tarball') * 2 + 1; + const packageTarballPath = resolve(argv[packageTarballValueIndex]); + const packageBytes = readPackedTarballSnapshot(packageTarballPath); + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-suite-snapshot-'), + ); + const snapshotPath = join(temporaryDirectory, 'package.tgz'); + try { + writeFileSync(snapshotPath, packageBytes, { mode: 0o600 }); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package tarball snapshot could not be prepared.'); + } + + const snapshottedArguments = [...argv]; + snapshottedArguments[packageTarballValueIndex] = snapshotPath; + return Object.freeze({ + argv: snapshottedArguments, + temporaryDirectory, + }); +} + +function runBoundedNode(scriptPath, args, failureMessage) { + const result = spawnSync(process.execPath, [scriptPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function runCoreNodePreservingError(args) { + const result = spawnSync(process.execPath, [coreRunnerPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + if (result.status !== 0) { + const message = result.stderr.trimEnd(); + throw new Error( + message.length > 0 ? message : 'Benchmark suite internal runner failed.', + ); + } + return result.stdout; +} + +function parseCoreManifest(stdout) { + let manifest; + try { + manifest = JSON.parse(stdout.trim()); + } catch { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.status !== 'completed' + ) { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + return manifest; +} + +function readPackedMarkdownModule(tarballPath) { + const result = spawnSync( + 'tar', + ['-xOzf', tarballPath, PACKED_MARKDOWN_MODULE_ENTRY], + { + cwd: repositoryRoot, + maxBuffer: MAX_MODULE_BYTES + 1, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }, + ); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > MAX_MODULE_BYTES + ) { + throw new Error('Benchmark suite packed Markdown module could not be read.'); + } + return result.stdout; +} + +function assertPackedSnapshotDigest(tarballPath, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(readPackedTarballSnapshot(tarballPath)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function htmlSerializationEvidenceArguments(values, modulePath, artifactSha256) { + return [ + '--input', + values['--html-input'], + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + artifactSha256, + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]; +} + +function runHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, htmlLegacyFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const legacyArguments = argumentsForFlags(values, legacyFlags); + let coreCompleted = false; + + try { + const coreStdout = runBoundedNode( + coreRunnerPath, + legacyArguments, + 'Benchmark suite internal runner failed.', + ); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + values['--module'], + values['--artifact-sha256'], + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runPackedHtmlSerializationSuite(argv) { + const flags = matchingFlags(argv); + const values = valuesForArguments(argv, flags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const coreFlags = flags.includes('--resulting-input') ? packedChangedFlags : packedFlags; + const coreArguments = argumentsForFlags(values, coreFlags); + const snapshotted = snapshotPackedArguments(coreArguments); + let coreCompleted = false; + + try { + const coreStdout = runCoreNodePreservingError(snapshotted.argv); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const snapshotValues = valuesForArguments(snapshotted.argv, coreFlags); + const snapshotTarballPath = snapshotValues['--package-tarball']; + const markdownModuleBytes = readPackedMarkdownModule(snapshotTarballPath); + const markdownModulePath = join( + snapshotted.temporaryDirectory, + 'cwl-markdown.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes, { mode: 0o600 }); + } catch { + throw new Error('Benchmark suite packed Markdown module could not be prepared.'); + } + const markdownArtifactSha256 = createHash('sha256') + .update(markdownModuleBytes) + .digest('hex'); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + markdownModulePath, + markdownArtifactSha256, + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + assertPackedSnapshotDigest( + snapshotTarballPath, + values['--package-sha256'], + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } +} + +function runExistingSuite(argv) { + const snapshotted = snapshotPackedArguments(argv); + + try { + const result = spawnSync( + process.execPath, + [coreRunnerPath, ...snapshotted.argv], + { + cwd: repositoryRoot, + stdio: 'inherit', + timeout: 600_000, + }, + ); + + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + + process.exitCode = result.status ?? 1; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } +} + +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); + assertPackedCorpusInputs(argv); + const expectedLegacySourceCommitSha = claimedLegacySourceCommitSha(argv); + if (expectedLegacySourceCommitSha !== undefined) { + assertCleanSourceCheckout(repositoryRoot, expectedLegacySourceCommitSha); + } + if (matchesArguments(argv, htmlLegacyFlags)) { + runHtmlSerializationSuite(argv); + return; + } + if (matchesArguments(argv, packedHtmlFlags) || matchesArguments(argv, packedHtmlChangedFlags)) { + runPackedHtmlSerializationSuite(argv); + return; + } + runExistingSuite(argv); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs new file mode 100644 index 000000000..c1cfdf79d --- /dev/null +++ b/benchmarks/source-checkout-provenance.mjs @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; + +const MAX_STATUS_BYTES = 1024 * 1024; + +function checkedOutHeadSha(repositoryRoot) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout identity could not be verified.', + ); + } + + return result.stdout.trim(); +} + +export function assertCleanSourceCheckout(repositoryRoot, expectedSourceCommitSha) { + const result = spawnSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout cleanliness could not be verified.', + ); + } + + if (result.stdout.length !== 0) { + throw new Error( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', + ); + } + + if ( + expectedSourceCommitSha !== undefined && + checkedOutHeadSha(repositoryRoot) !== expectedSourceCommitSha + ) { + throw new Error( + 'Benchmark suite source commit does not match checked-out HEAD.', + ); + } +} diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs new file mode 100644 index 000000000..ede369c11 --- /dev/null +++ b/benchmarks/summarize-samples.mjs @@ -0,0 +1,395 @@ +import { + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING_NOFOLLOW = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|transition-changed-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; +const UNITS = new Set(['ms', 'bytes']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const BENCHMARK_INPUT_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 4 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--output' || + argv[3].length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/summarize-samples.mjs --input --output ', + ); + } + return Object.freeze({ + inputPath: resolve(argv[1]), + outputDirectory: resolve(argv[3]), + }); +} + +function inspectSampleInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark sample input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectSampleInputPath(path); + if (pathMetadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } + if (pathMetadata === undefined || !pathMetadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + const bytes = Buffer.concat(chunks, totalBytes); + let text; + try { + text = UTF8_DECODER.decode(bytes); + } catch { + throw new Error('Benchmark sample input must be valid UTF-8 JSON.'); + } + let parsed; + try { + parsed = JSON.parse(text); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; + } finally { + closeSync(descriptor); + } +} + +function validateInput(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark sample input must be an object.'); + } + if (![1, 2, 3].includes(value.contractVersion)) { + throw new Error('Benchmark sample contractVersion must be 1, 2 or 3.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark benchmarkId is invalid.'); + } + const inputIdentity = {}; + if (value.contractVersion === 3) { + inputIdentity.inputSha256 = value.inputSha256; + if (value.benchmarkId.startsWith('transition-changed-evidence-')) { + inputIdentity.resultingInputSha256 = value.resultingInputSha256; + if (value.inputSha256 === value.resultingInputSha256) { + throw new Error('Benchmark changed-transition inputs must differ.'); + } + } + for (const digest of Object.values(inputIdentity)) { + if (typeof digest !== 'string' || !SHA256_PATTERN.test(digest)) { + throw new Error('Benchmark input identities must be lowercase SHA-256 digests.'); + } + } + } + const allowedKeys = new Set([...BENCHMARK_INPUT_KEYS, ...Object.keys(inputIdentity)]); + if (Object.keys(value).some((key) => !allowedKeys.has(key))) { + throw new Error('Benchmark sample input contains unsupported fields.'); + } + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { + throw new Error('Benchmark unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error( + 'Benchmark artifactSha256 must be a lowercase 64-character SHA-256 digest.', + ); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark documentProfile is invalid.'); + } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark sample profile must match documentProfile.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark referenceHardwareId is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES + ) { + throw new Error('Benchmark samples must be a non-empty bounded array.'); + } + if ( + value.samples.some( + (sample) => + typeof sample !== 'number' || !Number.isFinite(sample) || sample < 0, + ) + ) { + throw new Error('Benchmark samples must be finite non-negative numbers.'); + } + return Object.freeze({ + contractVersion: value.contractVersion, + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + ...inputIdentity, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + samples: Object.freeze([...value.samples]), + }); +} + +function nearestRank(sorted, percentile) { + const index = Math.ceil(percentile * sorted.length) - 1; + return sorted[index]; +} + +function summarize(input) { + const sorted = [...input.samples].sort((left, right) => left - right); + return Object.freeze({ + contractVersion: input.contractVersion, + benchmarkId: input.benchmarkId, + unit: input.unit, + sourceCommitSha: input.sourceCommitSha, + artifactSha256: input.artifactSha256, + ...(input.contractVersion === 3 ? { inputSha256: input.inputSha256 } : {}), + ...(input.resultingInputSha256 === undefined ? {} : { resultingInputSha256: input.resultingInputSha256 }), + documentProfile: input.documentProfile, + runtimeId: input.runtimeId, + referenceHardwareId: input.referenceHardwareId, + sampleCount: sorted.length, + percentileMethod: 'nearest-rank', + minimum: sorted[0], + p50: nearestRank(sorted, 0.5), + p75: nearestRank(sorted, 0.75), + p95: nearestRank(sorted, 0.95), + maximum: sorted.at(-1), + }); +} + +function formatSummary(summary) { + return [ + `contract_version=${summary.contractVersion}`, + `benchmark=${summary.benchmarkId}`, + `unit=${summary.unit}`, + `source_commit_sha=${summary.sourceCommitSha}`, + `artifact_sha256=${summary.artifactSha256}`, + ...(summary.contractVersion === 3 ? [`input_sha256=${summary.inputSha256}`] : []), + ...(summary.resultingInputSha256 === undefined ? [] : [`resulting_input_sha256=${summary.resultingInputSha256}`]), + `document_profile=${summary.documentProfile}`, + `runtime_id=${summary.runtimeId}`, + `reference_hardware_id=${summary.referenceHardwareId}`, + `samples=${summary.sampleCount}`, + `percentile_method=${summary.percentileMethod}`, + `minimum=${summary.minimum}`, + `p50=${summary.p50}`, + `p75=${summary.p75}`, + `p95=${summary.p95}`, + `maximum=${summary.maximum}`, + '', + ].join('\n'); +} + +function refersToSameFile(leftPath, rightPath) { + if (!existsSync(leftPath) || !existsSync(rightPath)) return false; + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +function assertRegularOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && !metadata.isFile()) { + throw new Error('Benchmark summary output paths must be regular files.'); + } +} + +function assertSingleLinkOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && metadata.nlink !== 1) { + throw new Error( + 'Benchmark summary output paths must not be multiply linked.', + ); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const current = inspectOutputDirectoryComponent(path); + if (current !== undefined) { + if (!current.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + return; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectoryComponent(path); + if (created === undefined || !created.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } +} + +function main() { + const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const summaryJsonPath = resolve(outputDirectory, 'summary.json'); + const summaryTextPath = resolve(outputDirectory, 'summary.txt'); + prepareOutputDirectory(outputDirectory); + if ( + inputPath === summaryJsonPath || + inputPath === summaryTextPath || + refersToSameFile(inputPath, summaryJsonPath) || + refersToSameFile(inputPath, summaryTextPath) + ) { + throw new Error('Benchmark output must not overwrite the sample input.'); + } + assertRegularOutputDestination(summaryJsonPath); + assertRegularOutputDestination(summaryTextPath); + if (refersToSameFile(summaryJsonPath, summaryTextPath)) { + throw new Error('Benchmark summary outputs must be distinct files.'); + } + assertSingleLinkOutputDestination(summaryJsonPath); + assertSingleLinkOutputDestination(summaryTextPath); + const input = validateInput(readBoundedJson(inputPath)); + const summary = summarize(input); + writeFileSync( + summaryJsonPath, + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8', + ); + writeFileSync(summaryTextPath, formatSummary(summary), 'utf8'); +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark summary failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/docs/TRD.md b/docs/TRD.md index 80cec07ac..f8d2d1237 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -133,6 +133,17 @@ Release publication verifies one exact integrated protected source head, expecte Queued, cancelled, skipped-required, absent, stale-head, predecessor-head, status-only, author-only, or synthetic-merge evidence is not success. A commit status, automated model verdict, comment, formal review, merge authority, and external registry publication are distinct evidence classes. +Active PR performance research records the first operation invocation without an unrecorded latency warmup, with timing boundaries and generation-comparison limits documented in [`benchmarks/README.md`](../benchmarks/README.md). This harness does not establish buyer-workload latency or a supported document-size guarantee; a measurement-method change starts a new baseline rather than proving a product speedup. + +The active Markdown/HTML research producer rejects an acquisition when repeated +serialization of the same captured input returns different strings. Comparison +occurs after timing and retains the first measured output; no output content is +published in the failure. This consistency check cannot establish conversion +fidelity or repeatability from one sample. It starts a fresh measurement baseline +without changing the sample schema or published editor contract. + +The active research sample/summary contract marks input-bound first-invocation JavaScript latency evidence as version 3. It derives SHA-256 identities from the captured bounded input bytes before timing; changed transitions preserve ordered previous/resulting identities. The summarizer preserves those identities and the comparator rejects mismatched inputs even when profile labels match. Legacy versions 1 and 2 remain readable without backfilled identities; cross-version comparisons fail closed. Corpus and suite-inventory versions are independent; no editor API or supported-performance promise changes. See the research harness for the explicit synthetic autosave input representation and claim limits. + ## Security, privacy, and operability dependencies `SECURITY.md`, `docs/THREAT_MODEL.md`, `docs/TEST_STRATEGY.md`, `docs/OPERABILITY.md`, `docs/TRACEABILITY.md`, and the detailed ADR corpus are part of this technical contract. Root `SECURITY.md` is `implemented_on_protected_main` and is the normative private vulnerability-reporting/coordinated-disclosure policy. ADR 0017 records the durable decision, ownership boundary, claim limits, and recovery/supersession semantics without duplicating the policy text. diff --git a/docs/doctoring/performance-input-identity.md b/docs/doctoring/performance-input-identity.md new file mode 100644 index 000000000..21049963d --- /dev/null +++ b/docs/doctoring/performance-input-identity.md @@ -0,0 +1,88 @@ +# Captured-input identity for performance evidence + +Status: Research only; active PR #379, not protected-main implementation +Date: 2026-09-06 +Owner: [performance support-envelope issue #375](https://github.com/ContextualWisdomLab/inkspan/issues/375) + +## Problem and decision + +A document profile alone does not identify the workload. The direct latency +commands accepted different input files under the same profile while their +receipts identified only the source, artifact, runtime and reference hardware. +Two such summaries could therefore appear comparable despite measuring different +documents. The locked synthetic suite already checked its fixtures; this defect +was at the direct producer/summary/comparison boundary. + +Version 3 derives an input SHA-256 from the already bounded read before calling +the measured operation. Changed transitions retain ordered previous and resulting +input identities. The summarizer preserves them and the comparator rejects a +mismatch before computing a regression or improvement. SHA-256 identifies bytes; +it does not establish workload realism, confidentiality, authenticity or runtime +correctness (National Institute of Standards and Technology [NIST], 2015). + +The implementation reuses the existing Node crypto module. Its hash API accepts +byte buffers directly, avoiding a decode/re-encode step that could discard a +UTF-8 byte-order mark (Node.js, n.d.). Hashing is outside the existing timer; +first-invocation accounting, sample counts and scenario assertions are unchanged. + +## Alternatives and failure prevention + +- Caller-supplied digests were rejected because they repeat the unverified claim. +- Hashing a second file read was rejected because the file can change after the + operation's input is captured. A regression test replaces it during module + loading and requires the original captured identity. +- Adding optional fields to version 2 was rejected because old readers could + compare evidence without enforcing input identity. Versions 1 and 2 keep their + original shapes and meanings; cross-version comparisons remain prohibited. +- No-file autosave hashes the actual prepared synthetic revision-evidence payload + (envelope plus revision). Hashing just its envelope would confuse this distinct + preparation with file-mode envelope input. A fixture captures the value passed + to the queue and verifies its identity. + +Malformed, missing, extra, legacy-backfilled and identical changed-transition +identities fail closed. No document contents or paths are added to receipts. +Digests are not anonymization: private or guessable inputs must not be published +merely because their content was replaced by a hash. + +## Evidence lineage + +Local retained evidence directory: +`/private/tmp/inkspan-input-identity-evidence.3O7Jac`. + +| Source revision | Experiment | Result and scope | +| --- | --- | --- | +| `69bdcc5c4d8ba757fb84d6e96c77fb2704cecf7c` | Three producer contracts | 6 failed identity checks, 29 passed; `producers-identity-red-69bd.log` | +| `ce48badc22ecf45cec8ff3f14e40a323deaae0fc` | Five-file chain RED | 27 failed, 58 passed; 7 failures were incorrect new directory-absence assertions, subsequently corrected to require no evidence files; `chain-identity-red.log` | +| `c4fba276b1ea8725a61f354bc0c3e26c58aa6c54` | Six focused files | 86 passed; `chain-identity-green.log` | +| `b866a1d5815bdbd2d6f3ba4091bd945b30f43cdc` | Captured/prepared input distinction | 1 synthetic-payload identity failure, 13 passed; `captured-input-red.log` | +| `c7d829d91057d2d307133f3038c818ab5f602283` | Same captured/prepared checks | 14 passed; `captured-input-green.log`; TypeScript check also passed | +| `c7d829d91057d2d307133f3038c818ab5f602283` | All 41 performance files, two workers | 158 passed, 8 failed, 1 worker RPC error; `all-performance-c7d829.log`, 325.70 seconds. Failures include unchanged test/subprocess deadlines and three null child exit statuses; this is not complete acceptance | + +These are harness correctness experiments, not latency improvements. Broader +acceptance must use the final exact head and retain every failed denominator. +Historical samples are neither rewritten nor backfilled. The immutable source +commits preserve the changes even if the local temporary evidence expires. + +Run the directly affected checks from the repository root: + +```sh +pnpm exec vitest run src/performanceMarkdownMeasurementContract.test.ts src/performanceRevisionMeasurementContract.test.ts src/performanceAutosaveMeasurementContract.test.ts src/performanceHtmlSerializationMeasurement.test.ts src/performanceMeasurementStatisticsContract.test.ts src/performanceRegressionComparatorContract.test.ts --maxWorkers=2 +``` + +## Remaining work and authority + +A real-workload corpus, fresh same-generation baselines, profiling and verified +optimizations remain necessary. This change does not prove the 20 ms target, +supported document sizes, browser interaction latency or release acceptance. +The corpus lock and suite inventory keep their independent version 1 contracts. +No editor API, host authority or accepted ADR changes. See the +[harness contract](../../benchmarks/README.md), [TRD](../TRD.md) and +[buyer gap](../product-technical-gap-baseline.md). + +## References + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). https://doi.org/10.6028/NIST.FIPS.180-4 + +Node.js. (n.d.). *Crypto: Hash update and digest*. Node.js v24 documentation. +Retrieved September 6, 2026, from https://nodejs.org/docs/latest-v24.x/api/crypto.html#hashupdatedata-inputencoding diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 18fd8749f..67578a8b8 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -129,7 +129,11 @@ After P0 blockers and independent non-conflicting lanes permit, prioritize: 1. provider-neutral comments, suggestions, review targets, and revision-scoped proposal acceptance without moving identity or persistence into Inkspan; 2. measured large-document latency/memory support envelopes with realistic - fixtures and deterministic failure behavior; + fixtures and deterministic failure behavior; the active research harness's + [first-invocation accounting, exact input identity and claim limits](../benchmarks/README.md) + do not close this buyer-workload gap. Reject inconsistent repeated conversion + results before publishing latency samples; consistency alone does not replace + workload-specific fidelity checks or realistic document evidence; 3. CJK IME, touch, and mobile editing assurance with truthful real-device versus emulated support claims; 4. an executable packed-package reference host proving SSR/hydration, native diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py new file mode 100644 index 000000000..a2a62e769 --- /dev/null +++ b/office/benchmarks/measure_render.py @@ -0,0 +1,337 @@ +"""Produce privacy-safe timing and peak-RSS evidence for canonical Office fixtures.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +OFFICE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = OFFICE_ROOT.parent +LOCK_PATH = REPOSITORY_ROOT / "benchmarks" / "office-fixtures.lock.json" +SOURCE_ROOT = OFFICE_ROOT / "src" +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from inkspan_office.safe_renderer import write_office_document # noqa: E402 + +MAX_ITERATIONS = 100 +MAX_TOKEN_CODE_UNITS = 128 +SAMPLE_TIMEOUT_SECONDS = 120 +TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") +GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") +SUPPORTED_FORMATS = {"docx", "xlsx", "pptx"} + + +class BenchmarkContractError(Exception): + """Raised when benchmark evidence cannot be produced safely and truthfully.""" + + +def _metadata_token(value: str, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) > MAX_TOKEN_CODE_UNITS + or not TOKEN_PATTERN.fullmatch(value) + ): + raise BenchmarkContractError(f"{label} must be a bounded metadata token") + return value + + +def _positive_iterations(value: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= MAX_ITERATIONS: + raise BenchmarkContractError("iterations must be between 1 and 100") + return value + + +def _load_fixture_contract(format_name: str, profile: str) -> tuple[int, str]: + try: + lock = json.loads(LOCK_PATH.read_text(encoding="utf-8")) + if lock.get("contractVersion") != 1 or lock.get("synthetic") is not True: + raise BenchmarkContractError("canonical Office fixture lock is invalid") + record = lock["formats"][format_name][profile] + expected_bytes = record["bytes"] + expected_sha256 = record["sha256"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("canonical Office fixture lock is invalid") from exc + if ( + not isinstance(expected_bytes, int) + or isinstance(expected_bytes, bool) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not SHA256_PATTERN.fullmatch(expected_sha256) + ): + raise BenchmarkContractError("canonical Office fixture lock is invalid") + return expected_bytes, expected_sha256 + + +def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256: str) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be opened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks: list[bytes] = [] + remaining = expected_bytes + while remaining: + chunk = os.read(descriptor, min(remaining, 1024 * 1024)) + if not chunk: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + payload = b"".join(chunks) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be read safely") from exc + finally: + os.close(descriptor) + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _read_exact_stdin_fixture(expected_bytes: int, expected_sha256: str) -> bytes: + payload = sys.stdin.buffer.read(expected_bytes + 1) + if len(payload) != expected_bytes or hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _source_sha() -> str: + status = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], + check=False, + capture_output=True, + text=True, + ) + if status.returncode != 0 or status.stdout: + raise BenchmarkContractError("benchmark checkout must be clean") + revision = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + sha = revision.stdout.strip() + if revision.returncode != 0 or not GIT_SHA_PATTERN.fullmatch(sha): + raise BenchmarkContractError("benchmark source revision could not be verified") + return sha + + +def _peak_rss_bytes() -> int: + try: + import resource + except ImportError as exc: # pragma: no cover - benchmark CI is POSIX + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") from exc + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if peak <= 0: + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") + if sys.platform == "darwin": + return int(peak) + return int(peak) * 1024 + + +def _child_measure(format_name: str, profile: str) -> int: + try: + profile = _metadata_token(profile, "fixture profile") + expected_bytes, expected_sha256 = _load_fixture_contract(format_name, profile) + payload_bytes = _read_exact_stdin_fixture(expected_bytes, expected_sha256) + payload = json.loads(payload_bytes.decode("utf-8")) + if not isinstance(payload, dict): + raise BenchmarkContractError("canonical Office fixture must contain an object") + if payload.get("format") != format_name: + raise BenchmarkContractError("canonical Office fixture format is inconsistent") + with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: + output_path = Path(directory) / f"render.{format_name}" + started = time.perf_counter_ns() + write_office_document(payload, output_path) + duration_ms = (time.perf_counter_ns() - started) / 1_000_000 + peak_rss_bytes = _peak_rss_bytes() + print( + json.dumps( + { + "format": format_name, + "durationMs": round(duration_ms, 6), + "peakRssBytes": peak_rss_bytes, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark render sample failed.", file=sys.stderr) + return 2 + + +def _run_sample(payload_bytes: bytes, format_name: str, profile: str) -> dict[str, Any]: + try: + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + "--child", + "--format", + format_name, + "--fixture-profile", + profile, + ], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + timeout=SAMPLE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise BenchmarkContractError("Office benchmark render sample timed out") from None + if completed.returncode != 0: + raise BenchmarkContractError("Office benchmark render sample failed") + try: + sample = json.loads(completed.stdout.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("Office benchmark render sample was invalid") from exc + if not isinstance(sample, dict): + raise BenchmarkContractError("Office benchmark render sample was invalid") + duration = sample.get("durationMs") + peak_rss = sample.get("peakRssBytes") + observed_format = sample.get("format") + if ( + observed_format != format_name + or not isinstance(duration, (int, float)) + or isinstance(duration, bool) + or not math.isfinite(duration) + or duration < 0 + or not isinstance(peak_rss, int) + or isinstance(peak_rss, bool) + or peak_rss <= 0 + ): + raise BenchmarkContractError("Office benchmark render sample was invalid") + return {"format": observed_format, "durationMs": duration, "peakRssBytes": peak_rss} + + +def _percentile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _summary(values: list[float], *, integral: bool) -> dict[str, int | float]: + result: dict[str, int | float] = {} + for name, quantile in (("p50", 0.50), ("p75", 0.75), ("p95", 0.95)): + value = _percentile(values, quantile) + result[name] = int(round(value)) if integral else round(value, 6) + maximum = max(values) + result["max"] = int(maximum) if integral else round(maximum, 6) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Measure a canonical synthetic Inkspan Office fixture") + parser.add_argument("--input", required=True, help="canonical generated Office fixture path") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--reference-hardware", required=True) + return parser + + +def _child_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Render one verified synthetic Inkspan Office fixture") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + return parser + + +def _main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + iterations = _positive_iterations(args.iterations) + profile = _metadata_token(args.fixture_profile, "fixture profile") + reference_hardware = _metadata_token(args.reference_hardware, "reference hardware") + expected_bytes, expected_sha256 = _load_fixture_contract(args.format, profile) + source_sha = _source_sha() + payload_bytes = _read_exact_regular_fixture( + Path(args.input), expected_bytes, expected_sha256 + ) + samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] + observed_source_sha = _source_sha() + if observed_source_sha != source_sha: + raise BenchmarkContractError("benchmark source revision changed during measurement") + duration_values = [float(sample["durationMs"]) for sample in samples] + rss_values = [float(sample["peakRssBytes"]) for sample in samples] + evidence = { + "contractVersion": 1, + "synthetic": True, + "operation": "office_render", + "fixtureId": f"{args.format}.{profile}", + "format": args.format, + "profile": profile, + "iterations": iterations, + "referenceHardware": reference_hardware, + "fixtureBytes": expected_bytes, + "fixtureSha256": expected_sha256, + "sourceSha": source_sha, + "runtime": { + "implementation": platform.python_implementation(), + "python": platform.python_version(), + "platform": sys.platform, + }, + "samples": samples, + "summary": { + "durationMs": _summary(duration_values, integral=False), + "peakRssBytes": _summary(rss_values, integral=True), + }, + } + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark measurement failed.", file=sys.stderr) + return 2 + + +def main() -> int: + """Run the benchmark command or its isolated one-render child process.""" + + if sys.argv[1:2] == ["--child"]: + args = _child_parser().parse_args(sys.argv[2:]) + return _child_measure(args.format, args.fixture_profile) + return _main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py new file mode 100644 index 000000000..b9e6c88f5 --- /dev/null +++ b/office/tests/test_performance_measurement.py @@ -0,0 +1,249 @@ +"""Contract tests for privacy-safe Office render performance evidence.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _measure_command(script: Path, request_path: Path, iterations: str) -> list[str]: + return [ + sys.executable, + str(script), + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + iterations, + "--reference-hardware", + "pytest-reference", + ] + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure a lock-verified synthetic fixture without copying its document body into evidence.""" + + request_path = tmp_path / "synthetic-docx-small.json" + request_bytes = _canonical_docx_small_fixture_bytes() + request_path.write_bytes(request_bytes) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "2"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + evidence = json.loads(completed.stdout) + assert evidence["contractVersion"] == 1 + assert evidence["synthetic"] is True + assert evidence["operation"] == "office_render" + assert evidence["fixtureId"] == "docx.small" + assert evidence["profile"] == "small" + assert evidence["format"] == "docx" + assert evidence["iterations"] == 2 + assert evidence["referenceHardware"] == "pytest-reference" + assert evidence["fixtureBytes"] == len(request_bytes) + assert len(evidence["fixtureSha256"]) == 64 + assert len(evidence["sourceSha"]) == 40 + assert evidence["runtime"]["implementation"] + assert evidence["runtime"]["python"] + assert evidence["runtime"]["platform"] + assert len(evidence["samples"]) == 2 + for sample in evidence["samples"]: + assert isinstance(sample["durationMs"], (int, float)) + assert sample["durationMs"] >= 0 + assert isinstance(sample["peakRssBytes"], int) + assert sample["peakRssBytes"] > 0 + for percentile in ("p50", "p75", "p95", "max"): + assert evidence["summary"]["durationMs"][percentile] >= 0 + assert evidence["summary"]["peakRssBytes"][percentile] > 0 + + combined_output = completed.stdout + completed.stderr + assert MULTILINGUAL_PARAGRAPH not in combined_output + assert str(request_path) not in combined_output + + +def test_measure_office_render_rejects_noncanonical_content_without_leaking_it( + tmp_path: Path, +) -> None: + """Do not label arbitrary document content as canonical synthetic benchmark evidence.""" + + sentinel = "PRIVATE-NONCANONICAL-DOCUMENT-SENTINEL" + request_path = tmp_path / "private-customer-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "private customer document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ), + encoding="utf-8", + ) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel not in completed.stderr + assert str(request_path) not in completed.stderr + + +def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( + tmp_path: Path, +) -> None: + """Reject impossible benchmark work before inspecting the caller-selected request path.""" + + request_path = tmp_path / "must-not-be-read.json" + request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1001"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "iterations must be between 1 and 100" in completed.stderr + assert str(request_path) not in completed.stderr + assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr + + +def test_office_render_sample_times_out_without_leaking_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bound a hung renderer child and normalize timeout failure without exposing document data.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_test") + benchmark_error = namespace["BenchmarkContractError"] + run_sample = namespace["_run_sample"] + observed_timeout: list[float | None] = [] + sentinel = b'PRIVATE-HUNG-RENDER-SENTINEL' + + def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + timeout = kwargs.get("timeout") + observed_timeout.append(timeout if isinstance(timeout, (int, float)) else None) + raise subprocess.TimeoutExpired(args[0] if args else "renderer", timeout or 0) + + monkeypatch.setattr(subprocess, "run", _timeout) + + with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: + run_sample(sentinel, "docx", "small") + + assert observed_timeout == [120] + assert sentinel.decode() not in str(exc_info.value) + + +def test_office_render_child_rejects_unverified_private_payload() -> None: + """Require the isolated child to re-verify canonical fixture identity before rendering.""" + + sentinel = "PRIVATE-DIRECT-CHILD-DOCUMENT-SENTINEL" + payload = json.dumps( + { + "format": "docx", + "title": "private direct child document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ).encode() + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--child", + "--format", + "docx", + "--fixture-profile", + "small", + ], + input=payload, + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + ) + + assert completed.returncode != 0 + assert b"input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel.encode() not in completed.stderr diff --git a/office/tests/test_performance_source_stability.py b/office/tests/test_performance_source_stability.py new file mode 100644 index 000000000..0c51cb6e3 --- /dev/null +++ b/office/tests/test_performance_source_stability.py @@ -0,0 +1,176 @@ +"""Source-stability contracts for Office benchmark provenance.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + + +def _initialize_clean_repository(repository: Path) -> Path: + repository.mkdir() + _git(repository, "init", "-q") + tracked = repository / "tracked.txt" + tracked.write_text("before\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "initial", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return tracked + + +def test_office_measurement_rejects_clean_source_revision_move_during_sampling( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Never emit benchmark evidence if a clean checkout advances while samples are acquired.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_source_stability") + main = namespace["_main"] + globals_ = main.__globals__ + + repository = tmp_path / "isolated-repository" + tracked = _initialize_clean_repository(repository) + globals_["REPOSITORY_ROOT"] = repository + + request_path = tmp_path / "synthetic-docx-small.json" + request_path.write_bytes(_canonical_docx_small_fixture_bytes()) + + def _move_revision( + _payload_bytes: bytes, + format_name: str, + _profile: str, + ) -> dict[str, object]: + tracked.write_text("after\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "advance", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return {"format": format_name, "durationMs": 1.0, "peakRssBytes": 1024} + + globals_["_run_sample"] = _move_revision + + result = main( + [ + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + "1", + "--reference-hardware", + "pytest-reference", + ] + ) + output = capsys.readouterr() + + assert result == 2 + assert output.out == "" + assert "benchmark source revision changed during measurement" in output.err + assert str(repository) not in output.err + assert str(request_path) not in output.err diff --git a/src/demoBundleChunking.test.ts b/src/demoBundleChunking.test.ts new file mode 100644 index 000000000..d6097e457 --- /dev/null +++ b/src/demoBundleChunking.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { demoVendorChunk } from '../vite.demo.chunking'; + +describe('demo bundle chunking contract', () => { + it('keeps major editor dependency families in deterministic vendor chunks', () => { + const pnpmPrefix = '/repo/node_modules/.pnpm/example/node_modules/'; + + expect(demoVendorChunk(`${pnpmPrefix}react-dom/client.js`)).toBe( + 'react-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/pm/state/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}prosemirror-state/dist/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/core/dist/index.js`)).toBe( + 'tiptap-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}marked/lib/marked.esm.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}turndown/lib/turndown.es.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}yjs/dist/yjs.mjs`)).toBe( + 'collaboration-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}lodash-es/lodash.js`)).toBe('vendor'); + }); + + it('leaves application modules to Rollup and normalizes Windows paths', () => { + expect(demoVendorChunk('/repo/demo/App.tsx')).toBeUndefined(); + expect(demoVendorChunk(String.raw`C:\repo\node_modules\react\index.js`)).toBe( + 'react-vendor', + ); + }); +}); diff --git a/src/performanceAutosaveMeasurementContract.test.ts b/src/performanceAutosaveMeasurementContract.test.ts new file mode 100644 index 000000000..7e0ac47e8 --- /dev/null +++ b/src/performanceAutosaveMeasurementContract.test.ts @@ -0,0 +1,264 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-autosave.mjs', +); +const sourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'a'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('autosave enqueue performance measurement', () => { + it('emits the canonical summarizable sample contract without persisting document content', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-measure-')); + const modulePath = join(directory, 'autosave.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const inputPath = join(directory, 'document-envelope.json'); + const outputPath = join(directory, 'samples.json'); + const moduleSource = [ + 'let queueCount = 0;', + 'export function createDocumentAutosaveQueue(options) {', + " if (++queueCount > 2) throw new Error('Unmeasured invocation');", + ' return Object.freeze({', + ' async enqueue(evidence) {', + " if (evidence.envelope.documentJson.content[0].content[0].text !== 'profile-bound synthetic input') throw new Error('wrong profile input');", + ' const result = await options.save(evidence);', + " if (result?.status !== 'saved') throw new Error('save failed');", + ' return Object.freeze({', + " status: 'saved',", + ' strongEntityTag: evidence.revision.strongEntityTag,', + ' });', + ' },', + ' resume() { return false; },', + ' async flush() { return Object.freeze({ state: \'idle\' }); },', + ' async close() { return Object.freeze({ state: \'closed\' }); },', + ' getSnapshot() { return Object.freeze({ state: \'idle\' }); },', + ' });', + '}', + '', + ].join('\n'); + const revisionModuleSource = [ + "import { createHash } from 'node:crypto';", + 'function freeze(value) { if (value && typeof value === \'object\') { for (const child of Object.values(value)) freeze(child); Object.freeze(value); } return value; }', + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " const envelope = freeze(JSON.parse(Buffer.from(source).toString('utf8')));", + " const digestHex = createHash('sha256').update(source).digest('hex');", + ' return Object.freeze({ envelope, revision: Object.freeze({ algorithm: \'SHA-256\', digestHex, strongEntityTag: `"sha256-${digestHex}"` }) });', + '}', + '', + ].join('\n'); + + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + writeFileSync( + inputPath, + '{"schemaId":"https://inkspan.io/schemas/document-envelope/v1","schemaVersion":1,"documentJson":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"profile-bound synthetic input"}]}]}}\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + inputPath, + '--revision-module', + revisionModulePath, + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputPath, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const outputText = readFileSync(outputPath, 'utf8'); + const evidence = JSON.parse(outputText) as { + contractVersion?: unknown; + benchmarkId?: unknown; + unit?: unknown; + sourceCommitSha?: unknown; + artifactSha256?: unknown; + documentProfile?: unknown; + runtimeId?: unknown; + referenceHardwareId?: unknown; + samples?: unknown[]; + }; + expect(evidence).toMatchObject({ + contractVersion: 3, + benchmarkId: 'autosave-enqueue-small', + unit: 'ms', + sourceCommitSha, + artifactSha256: sha256(moduleSource), + inputSha256: sha256(readFileSync(inputPath, 'utf8')), + documentProfile: 'small', + runtimeId, + referenceHardwareId, + }); + expect(Object.keys(evidence).sort()).toEqual( + [ + 'artifactSha256', + 'benchmarkId', + 'contractVersion', + 'documentProfile', + 'inputSha256', + 'referenceHardwareId', + 'runtimeId', + 'samples', + 'sourceCommitSha', + 'unit', + ].sort(), + ); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples?.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(outputText).not.toContain('Synthetic autosave benchmark document'); + expect(outputText).not.toContain(modulePath); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 20_000); + + it('measures active-revision coalescing without starting a second save', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-coalescing-')); + const modulePath = join(directory, 'autosave.mjs'); + const outputPath = join(directory, 'samples.json'); + const capturedInput = join(directory, 'captured-input.json'); + const moduleSource = `import { writeFileSync } from 'node:fs'; +let queueCount = 0; +export function createDocumentAutosaveQueue(options) { + if (++queueCount > 2) throw new Error('Unmeasured invocation'); + let active; + return { + enqueue(evidence) { + writeFileSync(${JSON.stringify(capturedInput)}, JSON.stringify(evidence)); + if (active) return active; + active = Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); + return active; + }, + async close() {}, + }; +}\n`; + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + const args = [ + measurementScript, + '--module', modulePath, + '--profile', 'small', + '--samples', '2', + '--source-commit-sha', sourceCommitSha, + '--artifact-sha256', sha256(moduleSource), + '--runtime-id', runtimeId, + '--reference-hardware-id', referenceHardwareId, + '--operation', 'coalescing', + '--output', outputPath, + ]; + + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toMatchObject({ + benchmarkId: 'autosave-coalescing-small', + inputSha256: sha256(readFileSync(capturedInput, 'utf8')), + samples: [expect.any(Number), expect.any(Number)], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('measures saved-result classification after the host save completes', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-autosave-commit-')); + const modulePath = join(directory, 'autosave.mjs'); + const outputPath = join(directory, 'samples.json'); + const moduleSource = `let queueCount = 0; +export function createDocumentAutosaveQueue(options) { + if (++queueCount > 2) throw new Error('Unmeasured invocation'); + return { + enqueue(evidence) { + return Promise.resolve(options.save(evidence)).then(() => ({ status: 'saved' })); + }, + async close() {}, + }; +}\n`; + try { + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync(process.execPath, [ + measurementScript, + '--module', modulePath, + '--profile', 'small', + '--samples', '2', + '--source-commit-sha', sourceCommitSha, + '--artifact-sha256', sha256(moduleSource), + '--runtime-id', runtimeId, + '--reference-hardware-id', referenceHardwareId, + '--operation', 'commit', + '--output', outputPath, + ], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toMatchObject({ + benchmarkId: 'autosave-commit-small', + samples: [expect.any(Number), expect.any(Number)], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts new file mode 100644 index 000000000..0555b3dba --- /dev/null +++ b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const cases = [ + { + name: 'Markdown', + script: 'benchmarks/measure-markdown.mjs', + expectedError: 'Markdown benchmark input must be a regular non-symlink file.', + }, + { + name: 'revision', + script: 'benchmarks/measure-revision-evidence.mjs', + expectedError: 'Revision benchmark input must be a regular non-symlink file.', + }, +] as const; + +describe('benchmark existing-output path privacy contract', () => { + for (const testCase of cases) { + it(`redacts ${testCase.name} input paths before existing-output alias checks`, () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-alias-path-')); + const privateSentinel = `tenant-private-${testCase.name.toLowerCase()}-input-parent`; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-input'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(output, '{}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + resolve(process.cwd(), testCase.script), + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(testCase.expectedError); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + } +}); diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts new file mode 100644 index 000000000..cc24431b1 --- /dev/null +++ b/src/performanceCorpusContract.test.ts @@ -0,0 +1,170 @@ +import { execFileSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkProfileLock { + readonly sections: number; + readonly bytes: number; + readonly sha256: string; + readonly htmlBytes: number; + readonly htmlSha256: string; + readonly envelopeBytes: number; + readonly envelopeSha256: string; + readonly changedEnvelopeBytes: number; + readonly changedEnvelopeSha256: string; +} + +interface BenchmarkCorpusLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly scripts: readonly [ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]; + readonly profiles: Readonly< + Record<'small' | 'medium' | 'large' | 'stress', BenchmarkProfileLock> + >; +} + +function runGenerator(outputDirectory: string): BenchmarkCorpusLock { + const script = resolve(process.cwd(), 'benchmarks/generate-corpus.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as BenchmarkCorpusLock; +} + +describe('deterministic synthetic performance corpus', () => { + it('reproduces the committed corpus lock exactly across independent runs', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/corpus.lock.json'), + 'utf8', + ), + ) as BenchmarkCorpusLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest.synthetic).toBe(true); + expect(firstManifest.scripts).toEqual([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]); + + for (const profile of ['small', 'medium', 'large', 'stress'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.md`)); + const secondBytes = readFileSync(join(second, `${profile}.md`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + const firstHtml = readFileSync(join(first, `${profile}.html`)); + const secondHtml = readFileSync(join(second, `${profile}.html`)); + expect(firstHtml.equals(secondHtml)).toBe(true); + expect(firstHtml.byteLength).toBe(expected.profiles[profile].htmlBytes); + expect(firstHtml.toString('utf8')).toContain( + `data-inkspan-benchmark-profile="${profile}"`, + ); + const firstEnvelope = readFileSync( + join(first, `${profile}.envelope.json`), + ); + const secondEnvelope = readFileSync( + join(second, `${profile}.envelope.json`), + ); + expect(firstEnvelope.equals(secondEnvelope)).toBe(true); + expect(firstEnvelope.byteLength).toBe( + expected.profiles[profile].envelopeBytes, + ); + const changedEnvelope = readFileSync(join(first, `${profile}.changed.envelope.json`)); + expect(changedEnvelope.equals(readFileSync(join(second, `${profile}.changed.envelope.json`)))).toBe(true); + expect(changedEnvelope.byteLength).toBe(expected.profiles[profile].changedEnvelopeBytes); + expect(expected.profiles[profile].changedEnvelopeSha256).not.toBe(expected.profiles[profile].envelopeSha256); + const edited = JSON.parse(changedEnvelope.toString('utf8')); + expect(edited.documentJson.content.pop()).toEqual({ + type: 'paragraph', content: [{ type: 'text', text: 'Synthetic appended edit.' }], + }); + expect(edited).toEqual(JSON.parse(firstEnvelope.toString('utf8'))); + expect( + JSON.parse(firstEnvelope.toString('utf8')).documentJson.content.map( + (node: { content?: readonly [{ text: string }] }) => + node.content?.[0].text ?? '', + ), + ).toEqual(firstBytes.toString('utf8').split('\n')); + } + + const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); + expect(smallBody).toContain( + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', + ); + for (const dimensions of ['1x1', '16x16', '64x64']) { + expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); + } + expect( + new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g)).size, + ).toBe(3); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of following a corpus output symlink', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.md'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'small.md')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of following a corpus output-directory symlink', () => { + const root = mkdtempSync( + join(tmpdir(), 'inkspan-benchmark-corpus-directory-symlink-'), + ); + const outputDirectory = join(root, 'output'); + const buyerOwnedDirectory = join(root, 'buyer-owned'); + const sentinelPath = join(buyerOwnedDirectory, 'sentinel.txt'); + try { + mkdirSync(buyerOwnedDirectory, { recursive: true }); + writeFileSync(sentinelPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(buyerOwnedDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(buyerOwnedDirectory)).toEqual(['sentinel.txt']); + expect(readFileSync(sentinelPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts new file mode 100644 index 000000000..f803b4152 --- /dev/null +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -0,0 +1,108 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('HTML serialization performance measurement', () => { + it('measures packed htmlToMarkdown without falling back to markdownToHtml', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-html-serialization-')); + const input = join(root, 'small.html'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(input, '

Synthetic benchmark fixture

\n', 'utf8'); + writeFileSync( + modulePath, + [ + 'let measuredCalls = 0;', + "export function markdownToHtml() { throw new Error('wrong serialization direction'); }", + "export function htmlToMarkdown(source) { if (++measuredCalls > 2) throw new Error('Unmeasured invocation'); return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const evidence = JSON.parse(readFileSync(output, 'utf8')) as { + contractVersion: number; + benchmarkId: string; + unit: string; + inputSha256: string; + documentProfile: string; + samples: unknown[]; + }; + expect(evidence.contractVersion).toBe(3); + expect(evidence.inputSha256).toBe(sha256(input)); + expect(evidence.benchmarkId).toBe('html-serialization-small'); + expect(evidence.unit).toBe('ms'); + expect(evidence.documentProfile).toBe('small'); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts new file mode 100644 index 000000000..8679dae89 --- /dev/null +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -0,0 +1,153 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const currentRuntimeId = `node-${process.versions.node}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('single-command HTML serialization benchmark contract', () => { + it('measures HTML-to-Markdown serialization alongside Markdown and revision evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-html-suite-')); + const markdownInput = join(directory, 'document.md'); + const htmlInput = join(directory, 'document.html'); + const markdownModule = join(directory, 'markdown.mjs'); + const revisionInput = join(directory, 'document-envelope.json'); + const revisionModule = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const markdownModuleSource = [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'); + const revisionModuleSource = `const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`; + + try { + writeFileSync(markdownInput, '# Buyer benchmark\n', 'utf8'); + writeFileSync(htmlInput, '

Buyer benchmark

\n', 'utf8'); + writeFileSync(markdownModule, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInput, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModule, revisionModuleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInput, + '--html-input', + htmlInput, + '--module', + markdownModule, + '--revision-input', + revisionInput, + '--revision-module', + revisionModule, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + currentSourceCommitSha, + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + currentRuntimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const manifest = JSON.parse(result.stdout.trim()) as Record; + expect(manifest).toMatchObject({ + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + status: 'completed', + }); + + const samples = JSON.parse( + readFileSync( + join(outputDirectory, 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { + benchmarkId?: unknown; + documentProfile?: unknown; + samples?: unknown[]; + }; + expect(samples.benchmarkId).toBe('html-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.json', + ), + 'utf8', + ), + ) as { benchmarkId?: unknown }; + expect(summary.benchmarkId).toBe('html-serialization-small'); + expect( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.txt', + ), + 'utf8', + ), + ).toContain('html-serialization-small'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 20_000); +}); diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts new file mode 100644 index 000000000..38949a047 --- /dev/null +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -0,0 +1,336 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 3; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); +const RUNTIME_ID = `node-${process.versions.node}`; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function measurementArguments( + input: string, + modulePath: string, + output: string, + artifactSha256 = existsSync(modulePath) + ? fileSha256(modulePath) + : FALLBACK_ARTIFACT_SHA256, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +describe('Markdown runtime measurement contract', () => { + it.each(['', '\uFEFF'])('writes bounded privacy-safe samples with UTF-8 prefix %j', (prefix) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeFileSync(input, `${prefix}# Buyer benchmark fixture\n\nSynthetic content only.\n`, 'utf8'); + writeFileSync( + modulePath, + "let measuredCalls = 0;\nexport function markdownToHtml(source) { if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); return `

${source.length}

`; }\n", + 'utf8', + ); + const artifactSha256 = fileSha256(modulePath); + + execFileSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath, artifactSha256), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 3, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256, + inputSha256: fileSha256(input), + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Buyer benchmark fixture'); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Synthetic content only'); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const summary = JSON.parse( + readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), + ) as { sampleCount: number; benchmarkId: string; unit: string }; + expect(summary).toMatchObject({ + contractVersion: 3, + sampleCount: 3, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + inputSha256: fileSha256(input), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('identifies captured bytes even if the file changes after reading', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-captured-input-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'serializer.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '\uFEFF# Captured input\n', 'utf8'); + const inputSha256 = fileSha256(input); + writeFileSync(modulePath, `import { writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(input)}, '# Later replacement\\n'); +export function markdownToHtml(source) { + if (source !== '# Captured input\\n') throw new Error('wrong captured input'); + return '

Captured input

'; +}\n`); + execFileSync(process.execPath, measurementArguments(input, modulePath, output), { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }); + expect(fileSha256(input)).not.toBe(inputSha256); + expect(JSON.parse(readFileSync(output, 'utf8'))).toMatchObject({ inputSha256, samples: [expect.any(Number), expect.any(Number), expect.any(Number)] }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + ['markdown-to-html', 'markdownToHtml', 1], + ['markdown-to-html', 'markdownToHtml', 3], + ['html-to-markdown', 'htmlToMarkdown', 1], + ['html-to-markdown', 'htmlToMarkdown', 3], + ] as const)('rejects %s non-string result from %s on invocation %i', (operation, exportName, invalidCall) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-serialization-invalid-result-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, + `let calls = 0;\nexport function ${exportName}() { return ++calls === ${invalidCall} ? { privateContent: 'never publish this' } : ''; }\n`, + 'utf8'); + const args = measurementArguments(input, modulePath, samplesPath); + args.splice(5, 0, '--operation', operation); + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(`Measured ${exportName}() must return a string.`); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + ['markdown-to-html', 'markdownToHtml', 2], + ['markdown-to-html', 'markdownToHtml', 3], + ['html-to-markdown', 'htmlToMarkdown', 2], + ['html-to-markdown', 'htmlToMarkdown', 3], + ] as const)('rejects changing %s output from %s on invocation %i', (operation, exportName, changedCall) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-serialization-changing-result-')); + const input = join(root, 'input.md'); + const modulePath = join(root, 'serializer.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n'); + writeFileSync(modulePath, `let calls = 0; +export function ${exportName}() { return ++calls === ${changedCall} ? 'private changed output' : ''; }\n`); + const args = measurementArguments(input, modulePath, output); + args.splice(5, 0, '--operation', operation); + const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Measured serialization output changed for identical input.'); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the public serializer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must export markdownToHtml().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked document inputs before reading benchmark content', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-symlink-')); + const realInput = join(root, 'real.md'); + const input = join(root, 'linked.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(realInput, '# Synthetic\n', 'utf8'); + symlinkSync(realInput, input); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return source; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects an output path that aliases the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-output-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, modulePath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output must not overwrite the measured module.', + ); + expect(readFileSync(modulePath, 'utf8')).toBe(moduleSource); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires a file-backed module URL rather than network or package-name resolution', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); + const input = join(root, 'small.md'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments( + input, + 'https://example.invalid/markdown.mjs', + samplesPath, + ), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(samplesPath)).toBe(false); + expect(() => pathToFileURL(input)).not.toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts new file mode 100644 index 000000000..3e172ebb8 --- /dev/null +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -0,0 +1,151 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function runMeasurement( + moduleSource: string, + outputForRoot: (root: string) => string = (root) => join(root, 'samples.json'), +) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = outputForRoot(root); + writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('Markdown measurement error privacy contract', () => { + it('redacts exceptions raised while loading the measured module', () => { + const privateSentinel = 'private-import-sentinel-must-not-leak'; + const moduleSource = `throw new Error('${privateSentinel}');\nexport function markdownToHtml(value) { return value; }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module could not be loaded.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts exceptions raised by the measured serializer', () => { + const privateSentinel = 'private-serializer-sentinel-must-not-leak'; + const moduleSource = `export function markdownToHtml() { throw new Error('${privateSentinel}'); }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured markdownToHtml() execution failed.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output path traversal fails', () => { + const privateSentinel = 'private-output-sentinel-must-not-leak'; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const { root, output, result } = runMeasurement(moduleSource, (testRoot) => { + const blockedParent = join(testRoot, privateSentinel); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + return join(blockedParent, 'samples.json'); + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const privateSentinel = `private-markdown-publication-${process.pid}`; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const blockedOutput = join('/sys', privateSentinel, 'samples.json'); + const { root, output, result } = runMeasurement( + moduleSource, + () => blockedOutput, + ); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts new file mode 100644 index 000000000..748e94a19 --- /dev/null +++ b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +describe('Markdown measurement input error privacy contract', () => { + it('redacts private filesystem details when input traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-input-privacy-')); + const privateSentinel = 'private-input-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 000000000..352e7f4df --- /dev/null +++ b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,65 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +describe('Markdown measurement module path error privacy contract', () => { + it('redacts private filesystem details when module path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-privacy-')); + const privateSentinel = 'private-module-sentinel-must-not-leak'; + const input = join(root, 'document.md'); + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'measured.mjs'); + const output = join(root, 'samples.json'); + + writeFileSync(input, '# bounded\n', 'utf8'); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + '0'.repeat(64), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 000000000..cc60c3cba --- /dev/null +++ b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,76 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('Markdown benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-output-ancestor-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source.length}

`; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementProvenanceContract.test.ts b/src/performanceMarkdownMeasurementProvenanceContract.test.ts new file mode 100644 index 000000000..dea6bb441 --- /dev/null +++ b/src/performanceMarkdownMeasurementProvenanceContract.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement artifact provenance', () => { + it('rejects caller metadata that does not match the exact measured module bytes', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-provenance-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'f'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownModuleUrlContract.test.ts b/src/performanceMarkdownModuleUrlContract.test.ts new file mode 100644 index 000000000..af022d475 --- /dev/null +++ b/src/performanceMarkdownModuleUrlContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement module URL authority', () => { + it('rejects file URLs with a non-local host before loading the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-url-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + const artifactSha256 = createHash('sha256') + .update(readFileSync(modulePath)) + .digest('hex'); + const nonLocalFileUrl = pathToFileURL(modulePath); + nonLocalFileUrl.hostname = 'example.invalid'; + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + nonLocalFileUrl.href, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts new file mode 100644 index 000000000..895732ad1 --- /dev/null +++ b/src/performanceMeasurementOutputContract.test.ts @@ -0,0 +1,199 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +function writeValidInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output contract', () => { + it('rejects an invalid second destination before publishing the first summary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(summaryText, { recursive: true }); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(summaryJson)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects a dangling summary symlink before it can create the symlink target', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-symlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const escapedTarget = join(root, 'escaped-summary.json'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + symlinkSync(escapedTarget, summaryJson); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(escapedTarget)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects two output paths that alias the same regular file before publication', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-alias-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + writeFileSync(summaryJson, 'sentinel', 'utf8'); + linkSync(summaryJson, summaryText); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary outputs must be distinct files.', + ); + expect(readFileSync(summaryJson, 'utf8')).toBe('sentinel'); + expect(readFileSync(summaryText, 'utf8')).toBe('sentinel'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-input-fifo-')); + const input = join(root, 'samples.pipe'); + const output = join(root, 'output'); + try { + const mkfifo = spawnSync('mkfifo', [input], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 1000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when the output directory cannot be prepared', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-privacy-')); + const input = join(root, 'samples.json'); + const privateSentinel = 'private-summary-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const output = join(blockedParent, 'output'); + try { + writeValidInput(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory could not be prepared.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts new file mode 100644 index 000000000..9f994c5ee --- /dev/null +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -0,0 +1,96 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +const validInput = { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], +} as const; + +function runSummary(inputValue: object) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + writeFileSync(input, `${JSON.stringify(inputValue)}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('benchmark evidence privacy contract', () => { + it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { + const { root, output, result } = runSummary({ + ...validInput, + prompt: 'must-not-enter-benchmark-evidence', + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input contains unsupported fields.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + [ + 'benchmarkId', + 'tenant-acme-large', + 'Benchmark benchmarkId is invalid.', + ], + ['unit', 'tenant-acme', 'Benchmark unit is invalid.'], + ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], + [ + 'referenceHardwareId', + 'tenant-acme', + 'Benchmark referenceHardwareId is invalid.', + ], + ])( + 'rejects caller-controlled %s values that could launder private identifiers into evidence', + (field, value, expectedError) => { + const { root, output, result } = runSummary({ + ...validInput, + [field]: value, + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedError); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts new file mode 100644 index 000000000..ffc4d283c --- /dev/null +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -0,0 +1,183 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + linkSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const markdownScript = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const revisionScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const sourceCommitSha = execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; + +function sha256(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +function commonArguments( + input: string, + module: string, + moduleSha256: string, + output: string, +): string[] { + return [ + '--input', + input, + '--module', + module, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + moduleSha256, + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ]; +} + +function expectOutputPreservedFailure( + script: string, + args: string[], + sentinel: string, + expectedMessage: string, +): void { + const originalSentinel = readFileSync(sentinel, 'utf8'); + const result = spawnSync(process.execPath, [script, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedMessage); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); +} + +describe('benchmark producer output immutability', () => { + it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Markdown benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-hardlink-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Revision benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before Markdown measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-existing-output-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Markdown benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-existing-output-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Revision benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts new file mode 100644 index 000000000..865d993b5 --- /dev/null +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const mismatchedSourceCommitSha = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); +const activeRuntimeId = `node-${process.versions.node}`; +const mismatchedRuntimeId = + activeRuntimeId === 'node-99.99.99' ? 'node-98.98.98' : 'node-99.99.99'; +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function markdownInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Provenance fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +function revisionInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Provenance fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +describe('direct benchmark producer source/runtime provenance', () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied source SHA that is not the checked-out HEAD for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-')); + try { + const { result, output } = invoke( + root, + mismatchedSourceCommitSha, + activeRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied runtime ID that is not the active Node runtime for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-runtime-')); + try { + const { result, output } = invoke( + root, + currentSourceCommitSha, + mismatchedRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts new file mode 100644 index 000000000..12cfdd9e1 --- /dev/null +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -0,0 +1,169 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const firstSourceSha = 'a'.repeat(40); +const movedSourceSha = 'b'.repeat(40); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'c'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function movingGitEnvironment(root: string): NodeJS.ProcessEnv { + const fakeBin = join(root, 'fake-bin'); + const statePath = join(root, 'git-invocations.txt'); + const fakeGit = join(fakeBin, 'git'); + const script = `#!/usr/bin/env node +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +const state = process.env.INKSPAN_FAKE_GIT_STATE; +const first = process.env.INKSPAN_FAKE_GIT_FIRST_SHA; +const moved = process.env.INKSPAN_FAKE_GIT_MOVED_SHA; +const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; +writeFileSync(state, String(count + 1), 'utf8'); +process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); +`; + mkdirSync(fakeBin, { recursive: true }); + writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); + chmodSync(fakeGit, 0o755); + return { + ...process.env, + PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_FAKE_GIT_STATE: statePath, + INKSPAN_FAKE_GIT_FIRST_SHA: firstSourceSha, + INKSPAN_FAKE_GIT_MOVED_SHA: movedSourceSha, + }; +} + +function markdownInvocation(root: string) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Source movement fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +function revisionInvocation(root: string) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'d'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Source movement fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +describe.skipIf(process.platform === 'win32')( + 'direct benchmark producer source stability', + () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects %s evidence when checked-out HEAD moves during sample acquisition', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-move-')); + try { + const { output, result } = invoke(root); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + }, +); diff --git a/src/performanceMeasurementStatisticsAncestorSymlink.test.ts b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts new file mode 100644 index 000000000..aae380be6 --- /dev/null +++ b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output path ancestry', () => { + it('fails closed before writing through a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-ancestor-')); + const input = join(root, 'samples.json'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + writeInput(input); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'summary.json'))).toBe(false); + expect(existsSync(join(outside, 'nested-output', 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts new file mode 100644 index 000000000..33be40b63 --- /dev/null +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -0,0 +1,397 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSummary { + readonly contractVersion: 1 | 2 | 3; + readonly benchmarkId: string; + readonly unit: string; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly sampleCount: number; + readonly percentileMethod: 'nearest-rank'; + readonly minimum: number; + readonly p50: number; + readonly p75: number; + readonly p95: number; + readonly maximum: number; +} + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +function writeInput(path: string, samples: readonly number[], contractVersion: unknown = 1): void { + writeFileSync( + path, + `${JSON.stringify( + { + contractVersion, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + ...(contractVersion === 3 ? { inputSha256: 'd'.repeat(64) } : {}), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummary { + execFileSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return JSON.parse( + readFileSync(join(outputDirectory, 'summary.json'), 'utf8'), + ) as BenchmarkSummary; +} + +describe('deterministic benchmark sample statistics', () => { + it.each([1, 2, 3] as const)('preserves generation %i in reproducible JSON and human-readable summaries', (contractVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); + const input = join(root, 'samples.json'); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + writeInput(input, [20, 10, 40, 30, 50], contractVersion); + + const firstSummary = runSummary(input, first); + const secondSummary = runSummary(input, second); + expect(firstSummary).toEqual(secondSummary); + expect(firstSummary).toEqual({ + contractVersion, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + ...(contractVersion === 3 ? { inputSha256: 'd'.repeat(64) } : {}), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 5, + percentileMethod: 'nearest-rank', + minimum: 10, + p50: 30, + p75: 40, + p95: 50, + maximum: 50, + }); + + const expectedText = [ + `contract_version=${contractVersion}`, + 'benchmark=markdown-serialization-large', + 'unit=ms', + `source_commit_sha=${SOURCE_COMMIT_SHA}`, + `artifact_sha256=${ARTIFACT_SHA256}`, + ...(contractVersion === 3 ? [`input_sha256=${'d'.repeat(64)}`] : []), + 'document_profile=large', + 'runtime_id=node-22.18.0', + 'reference_hardware_id=github-actions-ubuntu-24.04-x64', + 'samples=5', + 'percentile_method=nearest-rank', + 'minimum=10', + 'p50=30', + 'p75=40', + 'p95=50', + 'maximum=50', + '', + ].join('\n'); + expect(readFileSync(join(first, 'summary.txt'), 'utf8')).toBe(expectedText); + expect(readFileSync(join(second, 'summary.txt'), 'utf8')).toBe(expectedText); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([0, 4, '2', null])('rejects unsupported measurement generation %s', (contractVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-generation-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], contractVersion); + const result = spawnSync(process.execPath, + [script, '--input', input, '--output', output], { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark sample contractVersion must be 1, 2 or 3.'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('preserves ordered changed-transition input identities in both receipts', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-inputs-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], 3); + const samples = JSON.parse(readFileSync(input, 'utf8')); + samples.benchmarkId = 'transition-changed-evidence-large'; + samples.resultingInputSha256 = 'e'.repeat(64); + writeFileSync(input, JSON.stringify(samples)); + expect(runSummary(input, output)).toMatchObject({ inputSha256: 'd'.repeat(64), resultingInputSha256: 'e'.repeat(64) }); + expect(readFileSync(join(output, 'summary.txt'), 'utf8')).toContain(`input_sha256=${'d'.repeat(64)}\nresulting_input_sha256=${'e'.repeat(64)}\n`); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { inputSha256: undefined }, { inputSha256: 'private-path' }, { inputSha256: 'A'.repeat(64) }, + { resultingInputSha256: 'e'.repeat(64) }, { contractVersion: 2 }, + { benchmarkId: 'transition-changed-evidence-large' }, + { benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: 'd'.repeat(64) }, + ])('rejects invalid input identity metadata %j without writing evidence', (overrides) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-input-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, 2, 3], 3); + writeFileSync(input, JSON.stringify({ ...JSON.parse(readFileSync(input, 'utf8')), ...overrides })); + const result = spawnSync(process.execPath, [script, '--input', input, '--output', output], { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).not.toContain('private-path'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed when immutable provenance metadata is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on invalid measurement samples without coercion', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, -1, 3]); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark samples must be finite non-negative numbers.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects obviously oversized sample input before whole-file reads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-size-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const preload = join(root, 'reject-whole-file-read.mjs'); + try { + writeFileSync(input, '', 'utf8'); + truncateSync(input, 16 * 1024 * 1024 + 1); + writeFileSync( + preload, + `import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +const originalReadFileSync = fs.readFileSync.bind(fs); +const blockedInputPath = ${JSON.stringify(input)}; +fs.readFileSync = (path, ...argumentsList) => { + if (path === blockedInputPath) { + throw new Error('benchmark whole-file read sentinel'); + } + return originalReadFileSync(path, ...argumentsList); +}; +syncBuiltinESMExports(); +`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + '--import', + pathToFileURL(preload).href, + script, + '--input', + input, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input exceeds the supported size.', + ); + expect(result.stderr).not.toContain('benchmark whole-file read sentinel'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('refuses to overwrite the measurement input with generated evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); + const input = join(root, 'summary.json'); + const output = root; + try { + writeInput(input, [10, 20, 30]); + const originalInput = readFileSync(input, 'utf8'); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(root, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('refuses a hard-linked output alias without mutating source evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(output, { recursive: true }); + linkSync(input, summaryJson); + const originalInput = readFileSync(input, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before writing summaries through a symlink output directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-symlink-')); + const input = join(root, 'samples.json'); + const target = join(root, 'outside-target'); + const output = join(root, 'output-link'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(target); + symlinkSync(target, output, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(target, 'summary.json'))).toBe(false); + expect(existsSync(join(target, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsInputSymlink.test.ts b/src/performanceMeasurementStatisticsInputSymlink.test.ts new file mode 100644 index 000000000..0c481d536 --- /dev/null +++ b/src/performanceMeasurementStatisticsInputSymlink.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function validSamples(): string { + return `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`; +} + +describe('benchmark summary sample input file authority', () => { + it('fails closed instead of following a symlinked sample input', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-symlink-')); + const target = join(root, 'private-samples.json'); + const alias = join(root, 'samples.json'); + const output = join(root, 'summary'); + try { + writeFileSync(target, validSamples(), 'utf8'); + symlinkSync(target, alias, process.platform === 'win32' ? 'file' : undefined); + + const result = spawnSync( + process.execPath, + [script, '--input', alias, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsOutputHardlink.test.ts b/src/performanceMeasurementStatisticsOutputHardlink.test.ts new file mode 100644 index 000000000..75e18c002 --- /dev/null +++ b/src/performanceMeasurementStatisticsOutputHardlink.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'small', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output hard-link safety', () => { + it('fails closed before truncating an unrelated hard-linked output target', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-output-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const sentinel = join(root, 'buyer-owned.txt'); + const summaryJson = join(output, 'summary.json'); + + try { + writeInput(input); + mkdirSync(output); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, summaryJson); + const originalSentinel = readFileSync(sentinel, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must not be multiply linked.', + ); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts new file mode 100644 index 000000000..5ea36b6df --- /dev/null +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -0,0 +1,70 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/run-current-suite.mjs'); + +describe('benchmark suite output path ancestry', () => { + it('fails closed before preparing an output beneath a symlinked ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-ancestor-')); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + join(root, 'unused.md'), + '--module', + join(root, 'unused.mjs'), + '--revision-input', + join(root, 'unused-envelope.json'), + '--revision-module', + join(root, 'unused-revision.mjs'), + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--revision-artifact-sha256', + 'c'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementUtf8Contract.test.ts b/src/performanceMeasurementUtf8Contract.test.ts new file mode 100644 index 000000000..45ceec9f1 --- /dev/null +++ b/src/performanceMeasurementUtf8Contract.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence UTF-8 contract', () => { + it('rejects malformed UTF-8 before parsing or generating summary evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-utf8-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + const prefix = [ + '{"contractVersion":1,', + '"benchmarkId":"markdown-serialization-large",', + '"unit":"ms",', + `"sourceCommitSha":"${SOURCE_COMMIT_SHA}",`, + `"artifactSha256":"${ARTIFACT_SHA256}",`, + '"documentProfile":"large",', + '"runtimeId":"node-22.18.0",', + '"referenceHardwareId":"github-actions-ubuntu-24.04-x64",', + '"samples":[1,2,3],', + '"untrustedNote":"', + ].join(''); + writeFileSync( + input, + Buffer.concat([ + Buffer.from(prefix, 'utf8'), + Buffer.from([0x80]), + Buffer.from('"}\n', 'utf8'), + ]), + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be valid UTF-8 JSON.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts new file mode 100644 index 000000000..63f26dce6 --- /dev/null +++ b/src/performanceMemorySettlingContract.test.ts @@ -0,0 +1,213 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/analyze-memory-settling.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type MemoryEvidenceOverrides = Partial<{ + benchmarkId: string; + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + warmupSamples: number; + samples: number[]; +}>; + +function evidence(overrides: MemoryEvidenceOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + warmupSamples: 2, + samples: [900, 920, 1000, 1005, 995, 1010, 1015, 1008, 1012, 1010], + ...overrides, + }; +} + +function runAnalysis( + root: string, + input: ReturnType, + maxGrowthBytes: string, + windowSize = '3', +) { + const inputPath = join(root, 'memory-evidence.json'); + writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + windowSize, + '--max-growth-bytes', + maxGrowthBytes, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('retained-memory settling evidence contract', () => { + it('passes bounded settled growth using explicit warmup and comparison windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-pass-')); + try { + const result = runAnalysis(root, evidence(), '16'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 10, + warmupSamples: 2, + windowSize: 3, + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1010, + retainedGrowthBytes: 10, + maxGrowthBytes: 16, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a retained-memory growth breach while preserving the public receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-fail-')); + try { + const result = runAnalysis( + root, + evidence({ + samples: [900, 920, 1000, 1005, 995, 1100, 1120, 1110, 1130, 1140], + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toMatchObject({ + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1130, + retainedGrowthBytes: 130, + maxGrowthBytes: 50, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence that cannot supply two disjoint settled windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-short-')); + try { + const result = runAnalysis( + root, + evidence({ warmupSamples: 2, samples: [900, 920, 1000, 1005, 1010] }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts an input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-path-privacy-')); + try { + const privateMarker = 'tenant-private-memory-evidence-parent'; + const parentPath = join(root, privateMarker); + writeFileSync(parentPath, 'not-a-directory', 'utf8'); + const inputPath = join(parentPath, 'memory-evidence.json'); + const result = spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + '50', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); + try { + const result = runAnalysis( + root, + evidence({ + benchmarkId: 'editor-lifecycle-retained-memory-small', + documentProfile: 'large', + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before a precision-loss false green from an inexact even-window median', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-precision-')); + try { + const maxSafe = Number.MAX_SAFE_INTEGER; + const result = runAnalysis( + root, + evidence({ + warmupSamples: 0, + samples: [maxSafe - 1, maxSafe - 1, maxSafe - 1, maxSafe], + }), + '0.25', + '2', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling window median must be exactly representable.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts new file mode 100644 index 000000000..c55b36ddd --- /dev/null +++ b/src/performanceOfficeFixtureContract.test.ts @@ -0,0 +1,229 @@ +import { execFileSync } from 'node:child_process'; +import { + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface OfficeFixtureProfileLock { + readonly units: number; + readonly bytes: number; + readonly sha256: string; +} + +interface OfficeFixtureLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly formats: Readonly<{ + docx: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; + xlsx: Readonly< + Record<'small' | 'wide16384', OfficeFixtureProfileLock> + >; + pptx: Readonly< + Record<'small' | 'slide120', OfficeFixtureProfileLock> + >; + }>; +} + +function runGenerator(outputDirectory: string): OfficeFixtureLock { + const script = resolve(process.cwd(), 'benchmarks/generate-office-fixtures.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as OfficeFixtureLock; +} + +function expectDeterministicFixture( + first: string, + second: string, + fileName: string, + expectedBytes: number, +): void { + const firstBytes = readFileSync(join(first, fileName)); + const secondBytes = readFileSync(join(second, fileName)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expectedBytes); +} + +describe('deterministic synthetic Office performance fixtures', () => { + it('reproduces bounded DOCX, XLSX, and PPTX corpora including 100+ unit fixtures', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/office-fixtures.lock.json'), + 'utf8', + ), + ) as OfficeFixtureLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest).toEqual({ + contractVersion: 1, + synthetic: true, + formats: expected.formats, + }); + + for (const profile of ['small', 'page120'] as const) { + expectDeterministicFixture( + first, + second, + `docx-${profile}.json`, + expected.formats.docx[profile].bytes, + ); + } + for (const profile of ['small', 'wide16384'] as const) { + expectDeterministicFixture( + first, + second, + `xlsx-${profile}.json`, + expected.formats.xlsx[profile].bytes, + ); + } + for (const profile of ['small', 'slide120'] as const) { + expectDeterministicFixture( + first, + second, + `pptx-${profile}.json`, + expected.formats.pptx[profile].bytes, + ); + } + + const page120 = JSON.parse( + readFileSync(join(first, 'docx-page120.json'), 'utf8'), + ) as { + format: string; + blocks: Array<{ type: string; text?: string }>; + }; + expect(page120.format).toBe('docx'); + expect(page120.blocks.filter(({ type }) => type === 'heading')).toHaveLength(120); + expect(page120.blocks.filter(({ type }) => type === 'page_break')).toHaveLength(119); + expect(page120.blocks.some(({ text }) => text?.includes('한국어'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + + const wide = JSON.parse( + readFileSync(join(first, 'xlsx-wide16384.json'), 'utf8'), + ) as { + format: string; + sheets: Array<{ rows: unknown[][]; freeze_panes?: string }>; + }; + expect(wide.format).toBe('xlsx'); + expect(wide.sheets).toHaveLength(1); + expect(wide.sheets[0]?.rows[0]).toHaveLength(16_384); + expect(wide.sheets[0]?.freeze_panes).toBe('XFD1048576'); + + const slide120 = JSON.parse( + readFileSync(join(first, 'pptx-slide120.json'), 'utf8'), + ) as { + format: string; + slides: Array<{ title: string; bullets?: Array }>; + }; + expect(slide120.format).toBe('pptx'); + expect(slide120.slides).toHaveLength(120); + expect(slide120.slides.some(({ title }) => title.includes('한국어'))).toBe(true); + expect( + slide120.slides.some(({ bullets }) => + bullets?.some((bullet) => + typeof bullet === 'string' + ? bullet.includes('日本語') + : bullet.text.includes('日本語'), + ), + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of overwriting a file through an Office fixture output symlink', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of overwriting a multiply linked Office fixture output', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-hardlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'buyer-owned.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + linkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-dir-')); + const victimDirectory = join(root, 'buyer-owned'); + const outputDirectory = join(root, 'output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output ancestor', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-ancestor-')); + const victimDirectory = join(root, 'buyer-owned-parent'); + const linkedParent = join(root, 'linked-parent'); + const outputDirectory = join(linkedParent, 'nested-output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, linkedParent, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts new file mode 100644 index 000000000..b1f1c2285 --- /dev/null +++ b/src/performancePackedArtifactPathStability.test.ts @@ -0,0 +1,229 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'d'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture( + directory: string, + name: string, + markdownMarker: string, +): { + markdownModuleSha256: string; + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, `${name}-package-source`); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, `${name}-packed`); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + const markdownModule = `export function markdownToHtml(source) { return \`

\${source}

\`; }\n`; + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + markdownModule, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `const revision = { digestHex: '${'e'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (!provider) return { revision }; + const digestHex = Buffer.from(await provider.digest('SHA-256', source)).toString('hex'); + return { revision: { digestHex } }; +} +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-autosave.js'), + [ + 'export function createDocumentAutosaveQueue({ save }) {', + ' let active;', + ' return {', + ' enqueue(evidence) {', + " active ??= Promise.resolve(save(evidence)).then(() => ({ status: 'saved' }));", + ' return active;', + ' },', + ' async close() {},', + ' };', + '}', + '', + ].join('\n'), + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + markdownModuleSha256: sha256(markdownModule), + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function createTarInterpositionShim( + directory: string, + originalTarballPath: string, + adversarialTarballPath: string, +): { environment: NodeJS.ProcessEnv; originalBackupPath: string } { + const shimDirectory = join(directory, 'shim'); + mkdirSync(shimDirectory, { recursive: true }); + const originalBackupPath = join(directory, 'original-package-backup.tgz'); + copyFileSync(originalTarballPath, originalBackupPath); + const realTar = execFileSync('sh', ['-c', 'command -v tar'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + const shimPath = join(shimDirectory, 'tar'); + writeFileSync( + shimPath, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args[1];\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + 'utf8', + ); + chmodSync(shimPath, 0o755); + return { + originalBackupPath, + environment: { + ...process.env, + PATH: `${shimDirectory}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_TEST_REAL_TAR: realTar, + INKSPAN_TEST_ORIGINAL_TARBALL: originalTarballPath, + INKSPAN_TEST_ADVERSARIAL_TARBALL: adversarialTarballPath, + INKSPAN_TEST_ORIGINAL_BACKUP: originalBackupPath, + }, + }; +} + +describe('packed artifact benchmark path stability', () => { + it('measures the same tarball bytes whose package digest was verified', () => { + if (process.platform === 'win32') return; + + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-path-stability-')); + temporaryDirectories.push(directory); + const original = createPackedBenchmarkFixture(directory, 'original', 'verified'); + const adversarial = createPackedBenchmarkFixture(directory, 'adversarial', 'interposed'); + const { environment } = createTarInterpositionShim( + directory, + original.tarballPath, + adversarial.tarballPath, + ); + const corpusDirectory = join(directory, 'corpus'); + execFileSync( + process.execPath, + [ + resolve(repositoryRoot, 'benchmarks/generate-corpus.mjs'), + '--output', + corpusDirectory, + ], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const markdownInputPath = join(corpusDirectory, 'small.md'); + const revisionInputPath = join(corpusDirectory, 'small.envelope.json'); + const outputDirectory = join(directory, 'evidence'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + original.tarballPath, + '--package-sha256', + original.packageSha256, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--runtime-id', + activeRuntimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + const markdownEvidence = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { artifactSha256: string }; + expect(markdownEvidence.artifactSha256).toBe(original.markdownModuleSha256); + }, 20_000); +}); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts new file mode 100644 index 000000000..b2d9ffa5c --- /dev/null +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -0,0 +1,405 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture(directory: string): { + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, 'package-source'); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, 'packed'); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `import { createHash } from 'node:crypto'; +const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (!provider) return { revision }; + const digestHex = Buffer.from(await provider.digest('SHA-256', source)).toString('hex'); + return { revision: { digestHex } }; +} +export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, resulting) { + const previousRevision = { digestHex: createHash('sha256').update(previous).digest('hex') }; + const resultingRevision = { digestHex: createHash('sha256').update(resulting).digest('hex') }; + return { previousRevision, resultingRevision, changed: previousRevision.digestHex !== resultingRevision.digestHex }; +} +`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-autosave.js'), + [ + 'export function createDocumentAutosaveQueue({ save }) {', + ' let active;', + ' return {', + ' enqueue(evidence) {', + " active ??= Promise.resolve(save(evidence)).then(() => ({ status: 'saved' }));", + ' return active;', + ' },', + ' async close() {},', + ' };', + '}', + '', + ].join('\n'), + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function packedSuiteArguments(options: { + directory: string; + packageSha256: string; + runtimeId: string; + sourceCommitSha?: string; + tarballPath: string; + includeChangedTransition?: boolean; + includeHtml?: boolean; +}): string[] { + const corpusDirectory = join(options.directory, 'corpus'); + execFileSync( + process.execPath, + [ + resolve(repositoryRoot, 'benchmarks/generate-corpus.mjs'), + '--output', + corpusDirectory, + ], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const markdownInputPath = join(corpusDirectory, 'small.md'); + const htmlInputPath = join(corpusDirectory, 'small.html'); + const revisionInputPath = join(corpusDirectory, 'small.envelope.json'); + return [ + suitePath, + '--input', + markdownInputPath, + ...(options.includeHtml === false ? [] : ['--html-input', htmlInputPath]), + '--revision-input', + revisionInputPath, + '--package-tarball', + options.tarballPath, + '--package-sha256', + options.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + options.sourceCommitSha ?? sourceCommitSha, + '--runtime-id', + options.runtimeId, + '--reference-hardware-id', + referenceHardwareId, + ...(options.includeChangedTransition ? [ + '--resulting-input', join(corpusDirectory, 'small.changed.envelope.json'), + ] : []), + '--output', + join(options.directory, 'evidence'), + ]; +} + +describe('packed artifact benchmark suite contract', () => { + it.each([ + { includeChangedTransition: false, includeHtml: true }, + { includeChangedTransition: true, includeHtml: true }, + { includeChangedTransition: true, includeHtml: false }, + ])('binds packed evidence to run provenance ($includeChangedTransition, $includeHtml)', ({ + includeChangedTransition, includeHtml, + }) => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + includeChangedTransition, + includeHtml, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ + contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha, + runtimeId: activeRuntimeId, + referenceHardwareId, + packageName: '@contextualwisdomlab/cwl-editor', + packageVersion: '0.0.0-benchmark-fixture', + packageSha256: packed.packageSha256, + autosaveSamples: 'autosave/samples.json', + autosaveSummaryJson: 'autosave/summary/summary.json', + autosaveSummaryText: 'autosave/summary/summary.txt', + autosaveCoalescingSamples: 'autosave-coalescing/samples.json', + autosaveCoalescingSummaryJson: + 'autosave-coalescing/summary/summary.json', + autosaveCoalescingSummaryText: + 'autosave-coalescing/summary/summary.txt', + autosaveCommitSamples: 'autosave-commit/samples.json', + autosaveCommitSummaryJson: 'autosave-commit/summary/summary.json', + autosaveCommitSummaryText: 'autosave-commit/summary/summary.txt', + canonicalizationSamples: 'envelope-canonicalization/samples.json', + canonicalizationSummaryJson: + 'envelope-canonicalization/summary/summary.json', + canonicalizationSummaryText: + 'envelope-canonicalization/summary/summary.txt', + ...(includeHtml ? { htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt' } : {}), + status: 'completed', + }); + + const autosaveSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(autosaveSamples.benchmarkId).toBe('autosave-enqueue-small'); + expect(autosaveSamples).toMatchObject({ + contractVersion: 3, + inputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.envelope.json'))), + }); + expect(autosaveSamples.samples).toHaveLength(2); + + const coalescingSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave-coalescing', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(coalescingSamples.benchmarkId).toBe('autosave-coalescing-small'); + expect(coalescingSamples.samples).toHaveLength(2); + + const commitSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'autosave-commit', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(commitSamples.benchmarkId).toBe('autosave-commit-small'); + expect(commitSamples.samples).toHaveLength(2); + + const canonicalizationSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'envelope-canonicalization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(canonicalizationSamples.benchmarkId).toBe( + 'envelope-canonicalization-small', + ); + expect(canonicalizationSamples.samples).toHaveLength(2); + + if (includeChangedTransition) { + expect(JSON.parse(result.stdout)).toMatchObject({ + changedTransitionSamples: 'transition-changed/samples.json', + changedTransitionSummaryJson: 'transition-changed/summary/summary.json', + changedTransitionSummaryText: 'transition-changed/summary/summary.txt', + }); + const changedSamples = JSON.parse(readFileSync( + join(directory, 'evidence', 'transition-changed', 'samples.json'), 'utf8', + )); + expect(changedSamples.benchmarkId).toBe('transition-changed-evidence-small'); + expect(changedSamples).toMatchObject({ + contractVersion: 3, + inputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.envelope.json'))), + resultingInputSha256: sha256(readFileSync(join(directory, 'corpus', 'small.changed.envelope.json'))), + }); + expect(changedSamples.samples).toHaveLength(2); + } else { + expect(JSON.parse(result.stdout)).not.toHaveProperty('changedTransitionSamples'); + } + if (includeHtml) { + const htmlSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); + expect(htmlSamples.samples).toHaveLength(2); + } + }, 20_000); + + it('rejects a runtime identifier that does not match the active Node process', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-runtime-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: 'node-0.0.0', + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite runtime ID must match the active Node runtime.\n', + ); + }); + + it.each(['small.html', 'small.changed.envelope.json'])( + 'rejects %s when it does not match the committed corpus', (fileName) => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-corpus-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + const args = packedSuiteArguments({ + directory, + includeChangedTransition: true, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }); + writeFileSync(join(directory, 'corpus', fileName), '

tiny

\n'); + + const result = spawnSync(process.execPath, args, { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite inputs must match the committed corpus profile.\n', + ); + expect(existsSync(join(directory, 'evidence'))).toBe(false); + }); + + it('rejects source provenance that does not match the benchmark checkout', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-source-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + sourceCommitSha: '0'.repeat(40), + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit SHA must match the current benchmark checkout.\n', + ); + }); +}); diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts new file mode 100644 index 000000000..ed2ef7013 --- /dev/null +++ b/src/performanceRegressionComparatorContract.test.ts @@ -0,0 +1,405 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); +const CURRENT_ARTIFACT_SHA256 = 'c'.repeat(64); + +type SummaryOverrides = Partial<{ + contractVersion: number; + benchmarkId: string; + unit: string; + sourceCommitSha: string; + artifactSha256: string; + inputSha256: string; + resultingInputSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + sampleCount: number; + percentileMethod: string; + minimum: number; + p50: number; + p75: number; + p95: number; + maximum: number; +}>; + +function summary(overrides: SummaryOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + ...(overrides.contractVersion === 3 ? { + inputSha256: 'd'.repeat(64), + ...(overrides.benchmarkId?.startsWith('transition-changed-evidence-') + ? { resultingInputSha256: 'e'.repeat(64) } : {}), + } : {}), + ...overrides, + }; +} + +function runComparison( + root: string, + baseline: ReturnType, + current: ReturnType, + tolerancePercent: string, +) { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(baselinePath, `${JSON.stringify(baseline)}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(current)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + tolerancePercent, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('benchmark regression comparator contract', () => { + it.each([ + ['editor-input-large', 1], ['editor-input-large', 2], + ['transition-changed-evidence-large', 1], ['transition-changed-evidence-large', 2], + ['editor-input-large', 3], ['transition-changed-evidence-large', 3], + ] as const)( + 'compares exact-context %s generation %i evidence with an explicit tolerance', (benchmarkId, contractVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); + try { + const result = runComparison( + root, + summary({ benchmarkId, contractVersion }), + summary({ benchmarkId, contractVersion, artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + '5', + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion, + benchmarkId, + unit: 'ms', + ...(contractVersion === 3 ? { + inputSha256: 'd'.repeat(64), + ...(benchmarkId.startsWith('transition-changed-evidence-') + ? { resultingInputSha256: 'e'.repeat(64) } : {}), + } : {}), + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, + baselineValue: 100, + currentValue: 104, + maxRegressionPercent: 5, + regressionPercent: 4, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([[1, 2], [2, 1], [2, 3], [3, 2], [1, 3], [3, 1]])('rejects generation %i versus %i even with a generous tolerance', (baselineVersion, currentVersion) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-generation-')); + try { + const result = runComparison(root, + summary({ contractVersion: baselineVersion }), + summary({ contractVersion: currentVersion, artifactSha256: CURRENT_ARTIFACT_SHA256 }), '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summaries are not comparable: contractVersion differs.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each(['inputSha256', 'resultingInputSha256'] as const)('rejects a changed %s before reporting a speedup', (field) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-input-')); + try { + const baseline = summary({ contractVersion: 3, benchmarkId: 'transition-changed-evidence-large' }); + const current = { ...baseline, artifactSha256: CURRENT_ARTIFACT_SHA256, [field]: 'f'.repeat(64), p95: 90 }; + const result = runComparison(root, baseline, current, '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(`Benchmark summaries are not comparable: ${field} differs.`); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { contractVersion: 3, inputSha256: undefined }, + { contractVersion: 3, inputSha256: 'private-path' }, + { contractVersion: 3, inputSha256: 'A'.repeat(64) }, + { contractVersion: 3, resultingInputSha256: 'e'.repeat(64) }, + { contractVersion: 2, inputSha256: 'd'.repeat(64) }, + { contractVersion: 3, benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: undefined }, + { contractVersion: 3, benchmarkId: 'transition-changed-evidence-large', resultingInputSha256: 'd'.repeat(64) }, + ])('rejects invalid input identity metadata %j', (overrides) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-input-invalid-')); + try { + const result = runComparison(root, summary(overrides), summary({ ...overrides, artifactSha256: CURRENT_ARTIFACT_SHA256 }), '100'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).not.toContain('private-path'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects comparisons between changed and unchanged transition scenarios', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-compare-')); + try { + const result = runComparison(root, + summary({ benchmarkId: 'transition-evidence-large' }), + summary({ benchmarkId: 'transition-changed-evidence-large', artifactSha256: CURRENT_ARTIFACT_SHA256 }), '5'); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summaries are not comparable: benchmarkId differs.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a material unapproved regression without hiding the measured receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); + try { + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 106 }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, + baselineValue: 100, + currentValue: 106, + maxRegressionPercent: 5, + regressionPercent: 6, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects incomparable runtime or hardware evidence instead of laundering it through a tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-context-')); + try { + const result = runComparison( + root, + summary(), + summary({ referenceHardwareId: 'github-actions-ubuntu-22.04-x64' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summaries are not comparable: referenceHardwareId differs.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects private-looking units at the direct summary-comparison boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-unit-')); + try { + const result = runComparison( + root, + summary({ unit: 'tenant-acme' }), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, unit: 'tenant-acme' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summary unit is invalid.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires an explicit finite non-negative regression tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); + try { + const result = runComparison(root, summary(), summary(), '-1'); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked summary inputs instead of comparing mutable aliases', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-symlink-')); + const baselineTargetPath = join(root, 'baseline-target.json'); + const baselinePath = join(root, 'baseline-link.json'); + const currentPath = join(root, 'current.json'); + try { + writeFileSync(baselineTargetPath, `${JSON.stringify(summary())}\n`, 'utf8'); + symlinkSync(baselineTargetPath, baselinePath); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts a summary input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-path-privacy-')); + try { + const privateMarker = 'tenant-private-performance-baseline-parent'; + const privateParentPath = join(root, privateMarker); + const baselinePath = join(privateParentPath, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(privateParentPath, 'not-a-directory', 'utf8'); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fifo-')); + const baselinePath = join(root, 'baseline.pipe'); + const currentPath = join(root, 'current.json'); + try { + const mkfifo = spawnSync('mkfifo', [baselinePath], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + writeFileSync(currentPath, `${JSON.stringify(summary())}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8', timeout: 1000 }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionOverflowContract.test.ts b/src/performanceRegressionOverflowContract.test.ts new file mode 100644 index 000000000..2676c4586 --- /dev/null +++ b/src/performanceRegressionOverflowContract.test.ts @@ -0,0 +1,71 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(measurement: number, digestCharacter: string) { + return { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: digestCharacter.repeat(40), + artifactSha256: digestCharacter.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 1, + percentileMethod: 'nearest-rank', + minimum: measurement, + p50: measurement, + p75: measurement, + p95: measurement, + maximum: measurement, + }; +} + +describe('benchmark regression comparator overflow contract', () => { + it('fails closed instead of serializing an overflowing regression percentage as JSON null', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-overflow-')); + const baseline = join(root, 'baseline.json'); + const current = join(root, 'current.json'); + try { + writeFileSync( + baseline, + `${JSON.stringify(summary(1e-308, 'a'))}\n`, + 'utf8', + ); + writeFileSync( + current, + `${JSON.stringify(summary(1e308, 'b'))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baseline, + '--current', + current, + '--metric', + 'p95', + '--max-regression-percent', + '10', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionProfileConsistencyContract.test.ts b/src/performanceRegressionProfileConsistencyContract.test.ts new file mode 100644 index 000000000..bf7875275 --- /dev/null +++ b/src/performanceRegressionProfileConsistencyContract.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression profile consistency contract', () => { + it('rejects summaries whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-profile-')); + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + + try { + writeFileSync(baselinePath, `${JSON.stringify(summary('b'.repeat(64)))}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(summary('c'.repeat(64)))}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts new file mode 100644 index 000000000..55e90cda3 --- /dev/null +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -0,0 +1,135 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(sourceCommitSha: string, artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha, + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression provenance contract', () => { + it('binds each comparison receipt to both exact measured artifacts and the shared context', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const baselineSourceCommitSha = 'a'.repeat(40); + const currentSourceCommitSha = 'c'.repeat(40); + const baselineArtifactSha256 = 'b'.repeat(64); + const currentArtifactSha256 = 'd'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary(baselineSourceCommitSha, baselineArtifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary(currentSourceCommitSha, currentArtifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha, + baselineArtifactSha256, + currentSourceCommitSha, + currentArtifactSha256, + baselineValue: 100, + currentValue: 100, + maxRegressionPercent: 5, + regressionPercent: 0, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects a vacuous comparison when both summaries identify the same exact artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const artifactSha256 = 'b'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary('a'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary('c'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark summaries must identify distinct measured artifacts.\n', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts new file mode 100644 index 000000000..745bdf79b --- /dev/null +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -0,0 +1,505 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + linkSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 3; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function argumentsFor( + input: string, + modulePath: string, + output: string, + operation?: 'transition' | 'transition-changed' | 'canonicalization', + resultingInput?: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + fileSha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + ...(operation === undefined ? [] : ['--operation', operation]), + ...(resultingInput === undefined ? [] : ['--resulting-input', resultingInput]), + '--output', + output, + ]; +} + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision-evidence runtime measurement contract', () => { + it('writes privacy-safe revision samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'let measuredCalls = 0;\nexport async function createDocumentEnvelopeRevisionEvidenceBytes(source) { if (++measuredCalls > 3) throw new Error("Unmeasured invocation"); return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { + cwd: repositoryRoot, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 3, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain( + 'Synthetic benchmark content', + ); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + expect( + JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), + ).toMatchObject({ + sampleCount: 3, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + inputSha256: fileSha256(input), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('measures transition evidence through the same packed module boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `let measuredCalls = 0; +const revision = { digestHex: '${'d'.repeat(64)}' }; +export async function createDocumentEnvelopeTransitionEvidenceBytes() { if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, + 'utf8', + ); + + execFileSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'transition'), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ + benchmarkId: 'transition-evidence-large', + artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), + samples: expect.any(Array), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { operation: 'transition-changed', changed: true, equalDigests: false, valid: true }, + { operation: 'transition-changed', changed: false, equalDigests: true, valid: false }, + { operation: 'transition-changed', changed: true, equalDigests: true, valid: false }, + { operation: 'transition-changed', changed: false, equalDigests: false, valid: false }, + { operation: 'transition', changed: true, equalDigests: false, valid: false }, + { operation: 'transition', changed: false, equalDigests: false, valid: false }, + ] as const)('checks the $operation scenario oracle ($changed, $equalDigests)', ({ + operation, changed, equalDigests, valid, + }) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-changed-transition-')); + const input = join(root, 'previous.json'); + const resultingInput = join(root, 'resulting.json'); + const modulePath = join(root, 'revision.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(resultingInput, readFileSync(input, 'utf8').replace( + 'Synthetic benchmark content', 'Synthetic benchmark content with an edit', + )); + writeFileSync(modulePath, ` +let measuredCalls = 0; +export async function createDocumentEnvelopeTransitionEvidenceBytes(previous, resulting) { + if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); + if (previous.toString() !== ${JSON.stringify(readFileSync(input, 'utf8'))}) throw new Error('wrong previous source'); + if (resulting.toString() !== ${JSON.stringify(readFileSync(operation === 'transition' ? input : resultingInput, 'utf8'))}) throw new Error('wrong resulting source'); + return { previousRevision: { digestHex: '${'a'.repeat(64)}' }, resultingRevision: { digestHex: '${(equalDigests ? 'a' : 'b').repeat(64)}' }, changed: ${changed} }; +} +`); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, samplesPath, operation, + operation === 'transition-changed' ? resultingInput : undefined, + ), { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(valid ? 0 : 1); + expect(result.stdout).toBe(''); + expect(existsSync(samplesPath)).toBe(valid); + if (valid) { + const output = readFileSync(samplesPath, 'utf8'); + expect(JSON.parse(output)).toMatchObject({ + benchmarkId: 'transition-changed-evidence-large', + inputSha256: fileSha256(input), + resultingInputSha256: fileSha256(resultingInput), + samples: expect.any(Array), + }); + expect(JSON.parse(output).samples).toHaveLength(3); + expect(output).not.toContain('Synthetic benchmark content'); + expect(output).not.toContain(root); + execFileSync(process.execPath, [summaryScript, '--input', samplesPath, + '--output', join(root, 'summary')], { stdio: 'pipe' }); + } else { + expect(result.stderr.trim()).toBe('Measured revision-evidence result is invalid.'); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each(['missing', 'symlink', 'identical', 'overwrite', 'hardlink'] as const)( + 'rejects a %s resulting input without publishing samples or changing input', + (scenario) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-resulting-input-')); + const input = join(root, 'previous.json'); + const resultingInput = join(root, 'private-resulting.json'); + const modulePath = join(root, 'revision.mjs'); + let samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'throw new Error("module must not execute");'); + if (scenario === 'symlink') symlinkSync(input, resultingInput); + else if (scenario !== 'missing') { + writeFileSync(resultingInput, scenario === 'identical' + ? readFileSync(input) : 'private resulting source'); + } + if (scenario === 'overwrite') samplesPath = resultingInput; + if (scenario === 'hardlink') linkSync(resultingInput, samplesPath); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, samplesPath, 'transition-changed', resultingInput, + ), { cwd: repositoryRoot, encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + scenario === 'identical' + ? 'Changed-transition benchmark inputs must differ.' + : scenario === 'overwrite' || scenario === 'hardlink' + ? 'Revision benchmark output must not overwrite its input.' + : 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(root); + if (scenario === 'overwrite' || scenario === 'hardlink') { + expect(readFileSync(resultingInput, 'utf8')).toBe('private resulting source'); + } else expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each(['transition-changed', 'transition', 'canonicalization', undefined] as const)( + 'requires a resulting input only for changed transitions (%s)', (operation) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-arguments-')); + const input = join(root, 'input.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(modulePath, 'throw new Error("must not execute");'); + const result = spawnSync(process.execPath, argumentsFor( + input, modulePath, output, operation, + operation === 'transition-changed' ? undefined : input, + ), { encoding: 'utf8' }); + expect(result.status).toBe(1); + expect(result.stderr.trim()).toBe(operation === undefined + ? 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ' + : 'Only changed-transition measurement requires a resulting input.'); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it('isolates strict envelope canonicalization from digest-provider cost', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-canonicalization-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `let measuredCalls = 0; +export async function createDocumentEnvelopeRevisionEvidenceBytes(source, limits, provider) { + if (++measuredCalls > 3) throw new Error('Unmeasured invocation'); + const digest = await provider.digest('SHA-256', source); + return { revision: { digestHex: Buffer.from(digest).toString('hex') } }; +}\n`, + 'utf8', + ); + + execFileSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'canonicalization'), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + expect(JSON.parse(readFileSync(samplesPath, 'utf8'))).toMatchObject({ + benchmarkId: 'envelope-canonicalization-large', + artifactSha256: fileSha256(modulePath), + inputSha256: fileSha256(input), + samples: expect.any(Array), + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the revision API', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); + const input = join(root, 'small.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts hostile revision-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-revision-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return new Proxy({}, { get(_target, property) { if (property === 'revision') throw new Error('${privateSentinel}'); return undefined; } }); }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts hostile transition-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-transition-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-transition-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeTransitionEvidenceBytes() { const revision = new Proxy({}, { get() { throw new Error('${privateSentinel}'); } }); return { previousRevision: revision, resultingRevision: revision, changed: false }; }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath, 'transition'), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-output-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = 'private-revision-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const samplesPath = join(blockedParent, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-publication-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = `private-revision-publication-${process.pid}`; + const samplesPath = join('/sys', privateSentinel, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts new file mode 100644 index 000000000..fee613534 --- /dev/null +++ b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts @@ -0,0 +1,60 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +describe('revision benchmark input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-input-path-')); + const privateSentinel = 'tenant-private-revision-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-envelope.json'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 000000000..814f35c64 --- /dev/null +++ b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision benchmark module-path privacy contract', () => { + it('redacts filesystem details when module-path resolution crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-module-path-')); + const input = join(root, 'small.json'); + const privateSentinel = 'tenant-private-revision-module-parent'; + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'packed-revision-evidence.mjs'); + const output = join(root, 'samples.json'); + + try { + writeSyntheticEnvelope(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 000000000..48348e123 --- /dev/null +++ b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,84 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-revision-evidence.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('revision benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-output-ancestor-')); + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync( + input, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { type: 'doc', content: [] }, + }), + 'utf8', + ); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts new file mode 100644 index 000000000..7f2681739 --- /dev/null +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -0,0 +1,334 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const currentRuntimeId = `node-${process.versions.node}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function benchmarkArguments( + markdownInputPath: string, + markdownModulePath: string, + markdownArtifactSha256: string, + revisionInputPath: string, + revisionModulePath: string, + revisionArtifactSha256: string, + outputDirectory: string, + sourceCommitSha = currentSourceCommitSha, +): string[] { + return [ + suitePath, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + markdownArtifactSha256, + '--revision-artifact-sha256', + revisionArtifactSha256, + '--runtime-id', + currentRuntimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function writeBenchmarkInputs(directory: string): { + markdownArtifactSha256: string; + markdownInputPath: string; + markdownModulePath: string; + revisionArtifactSha256: string; + revisionInputPath: string; + revisionModulePath: string; +} { + const markdownInputPath = join(directory, 'input.md'); + const markdownModulePath = join(directory, 'markdown-measured.mjs'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionInputPath = join(directory, 'document-envelope.json'); + const revisionModulePath = join(directory, 'revision-measured.mjs'); + const revisionModuleSource = `const revision = { digestHex: '${'c'.repeat(64)}' }; +export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision }; } +export async function createDocumentEnvelopeTransitionEvidenceBytes() { return { previousRevision: revision, resultingRevision: revision, changed: false }; } +`; + + writeFileSync(markdownInputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + + return { + markdownArtifactSha256: createHash('sha256') + .update(markdownModuleSource) + .digest('hex'), + markdownInputPath, + markdownModulePath, + revisionArtifactSha256: createHash('sha256') + .update(revisionModuleSource) + .digest('hex'), + revisionInputPath, + revisionModulePath, + }; +} + +describe('single-command benchmark suite contract', () => { + it('measures and summarizes Markdown serialization and revision evidence with one command', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + + const output = execFileSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(JSON.parse(output.trim())).toEqual({ + contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha: currentSourceCommitSha, + runtimeId: currentRuntimeId, + referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + transitionSamples: 'transition/samples.json', + transitionSummaryJson: 'transition/summary/summary.json', + transitionSummaryText: 'transition/summary/summary.txt', + status: 'completed', + }); + + const markdownSamples = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(markdownSamples.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSamples.documentProfile).toBe('small'); + expect(markdownSamples.samples).toHaveLength(2); + + const markdownSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(markdownSummary.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('markdown-serialization-small'); + + const revisionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'revision', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(revisionSamples.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSamples.documentProfile).toBe('small'); + expect(revisionSamples.samples).toHaveLength(2); + + const revisionSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(revisionSummary.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('revision-evidence-small'); + + const transitionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'transition', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; samples?: unknown }; + expect(transitionSamples.benchmarkId).toBe('transition-evidence-small'); + expect(transitionSamples.samples).toHaveLength(2); + }, 20_000); + + it('rejects a claimed source commit that is not the checked-out HEAD', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-source-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const mismatchedCommit = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + mismatchedCommit, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(currentSourceCommitSha); + expect(result.stderr).not.toContain(mismatchedCommit); + expect(existsSync(outputDirectory)).toBe(false); + }); + + it('removes partial suite evidence when a downstream measurement fails', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const failingRevisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private benchmark failure'); }\n"; + writeFileSync( + inputs.revisionModulePath, + failingRevisionModuleSource, + 'utf8', + ); + const failingRevisionArtifactSha256 = createHash('sha256') + .update(failingRevisionModuleSource) + .digest('hex'); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + failingRevisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite revision measurement failed.\n', + ); + expect(existsSync(outputDirectory)).toBe(false); + }); + + it('fails closed before writing evidence through a symlink output directory', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const inputs = writeBenchmarkInputs(directory); + const actualOutputDirectory = join(directory, 'outside-target'); + const outputDirectory = join(directory, 'evidence-link'); + mkdirSync(actualOutputDirectory); + symlinkSync( + actualOutputDirectory, + outputDirectory, + process.platform === 'win32' ? 'junction' : 'dir', + ); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toBe( + 'Benchmark suite output directory must be a non-symlink directory.\n', + ); + expect(existsSync(join(actualOutputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'revision'))).toBe(false); + }, 20_000); +}); diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts new file mode 100644 index 000000000..34ab09d08 --- /dev/null +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -0,0 +1,115 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const helperUrl = pathToFileURL( + resolve(repositoryRoot, 'benchmarks/source-checkout-provenance.mjs'), +).href; +const temporaryDirectories: string[] = []; + +const probe = ` +import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; +try { + assertCleanSourceCheckout(process.argv[1], process.argv[2]); + process.stdout.write('clean\\n'); +} catch (error) { + process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); + process.exitCode = 1; +} +`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function createRepository(): string { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + temporaryDirectories.push(directory); + execFileSync('git', ['init', '--quiet'], { cwd: directory }); + execFileSync('git', ['config', 'user.email', 'test@example.invalid'], { + cwd: directory, + }); + execFileSync('git', ['config', 'user.name', 'Inkspan Test'], { + cwd: directory, + }); + writeFileSync(join(directory, 'tracked.txt'), 'committed\n'); + execFileSync('git', ['add', 'tracked.txt'], { cwd: directory }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture'], { cwd: directory }); + return directory; +} + +function headSha(directory: string): string { + return execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: directory, + encoding: 'utf8', + }).trim(); +} + +function probeCheckout(directory: string, expectedCommitSha = headSha(directory)) { + return spawnSync( + process.execPath, + ['--input-type=module', '--eval', probe, directory, expectedCommitSha], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); +} + +describe('benchmark source checkout provenance', () => { + it('accepts a clean source checkout at the claimed source commit', () => { + const directory = createRepository(); + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stdout).toBe('clean\n'); + expect(result.stderr).toBe(''); + }); + + it('rejects a clean checkout when the claimed source commit is not HEAD', () => { + const directory = createRepository(); + const actualHead = headSha(directory); + const mismatchedCommit = actualHead === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + const result = probeCheckout(directory, mismatchedCommit); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(actualHead); + expect(result.stderr).not.toContain(mismatchedCommit); + }); + + it('rejects untracked source state without disclosing paths', () => { + const directory = createRepository(); + const untrackedPath = join(directory, 'untracked-secret-name.txt'); + writeFileSync(untrackedPath, 'not part of the committed source\n'); + + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', + ); + expect(result.stderr).not.toContain(untrackedPath); + expect(result.stderr).not.toContain('untracked-secret-name.txt'); + }); +}); diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts new file mode 100644 index 000000000..063b0ce4b --- /dev/null +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -0,0 +1,179 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const coreSuitePath = resolve( + repositoryRoot, + 'benchmarks/run-current-suite-core.mjs', +); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function existingOutputArguments({ + suite, + markdownInputPath, + markdownModulePath, + revisionInputPath, + revisionModulePath, + markdownModuleSource, + revisionModuleSource, + outputDirectory, +}: { + suite: string; + markdownInputPath: string; + markdownModulePath: string; + revisionInputPath: string; + revisionModulePath: string; + markdownModuleSource: string; + revisionModuleSource: string; + outputDirectory: string; +}) { + return [ + suite, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function makeExistingOutputFixture(directory: string) { + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + return { + markdownInputPath, + revisionInputPath, + markdownModulePath, + revisionModulePath, + outputDirectory, + priorEvidencePath, + markdownModuleSource, + revisionModuleSource, + }; +} + +function expectExistingEvidenceUntouched({ + outputDirectory, + priorEvidencePath, +}: { + outputDirectory: string; + priorEvidencePath: string; +}) { + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); +} + +describe('benchmark suite existing-output atomicity', () => { + it('rejects an existing evidence directory before mutating prior evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: suitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expectExistingEvidenceUntouched(fixture); + }); + + it('rejects an output directory created after the wrapper preflight', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-core-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: coreSuitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expectExistingEvidenceUntouched(fixture); + }); +}); diff --git a/src/performanceSummaryInputPathPrivacyContract.test.ts b/src/performanceSummaryInputPathPrivacyContract.test.ts new file mode 100644 index 000000000..64ca60508 --- /dev/null +++ b/src/performanceSummaryInputPathPrivacyContract.test.ts @@ -0,0 +1,37 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const summarizer = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-path-')); + const privateSentinel = 'tenant-private-summary-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'samples.json'); + const output = join(root, 'summary'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [summarizer, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceSummaryProfileConsistencyContract.test.ts b/src/performanceSummaryProfileConsistencyContract.test.ts new file mode 100644 index 000000000..118346de3 --- /dev/null +++ b/src/performanceSummaryProfileConsistencyContract.test.ts @@ -0,0 +1,47 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary profile consistency contract', () => { + it('rejects sample evidence whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-profile-')); + const inputPath = join(root, 'samples.json'); + const outputDirectory = join(root, 'summary'); + + try { + writeFileSync( + inputPath, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [70, 80, 90, 100], + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceWorkflowContract.test.ts b/src/performanceWorkflowContract.test.ts new file mode 100644 index 000000000..f4404c8a3 --- /dev/null +++ b/src/performanceWorkflowContract.test.ts @@ -0,0 +1,71 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const workflowPath = resolve( + repositoryRoot, + '.github/workflows/performance-evidence.yml', +); + +describe('performance evidence workflow contract', () => { + it('runs a bounded exact-head packed-artifact benchmark on performance-relevant PRs', () => { + expect(existsSync(workflowPath)).toBe(true); + if (!existsSync(workflowPath)) return; + + const workflow = readFileSync(workflowPath, 'utf8'); + + expect(workflow).toContain('name: Performance Evidence'); + expect(workflow).toContain('pull_request:'); + expect(workflow).toContain("cron: '17 3 * * 1'"); + expect(workflow).toContain('workflow_dispatch:'); + expect(workflow).toContain('permissions:\n contents: read'); + expect(workflow).toContain( + "group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }}", + ); + expect(workflow).toContain('cancel-in-progress: true'); + expect(workflow).toContain('runs-on: ubuntu-24.04'); + expect(workflow).toContain('timeout-minutes: 20'); + expect(workflow).toContain( + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', + ); + expect(workflow).toContain( + 'ref: ${{ github.event.pull_request.head.sha || github.sha }}', + ); + expect(workflow).toContain('persist-credentials: false'); + expect(workflow).toContain('Verify exact checkout'); + expect(workflow).toContain('pnpm install --frozen-lockfile'); + expect(workflow).toContain('pnpm build'); + expect(workflow).toContain('pnpm pack --pack-destination'); + expect(workflow).toContain('node benchmarks/generate-corpus.mjs'); + expect(workflow).toContain( + '"profile":"small","samples":3', + ); + for (const profile of ['small', 'medium', 'large', 'stress']) { + expect(workflow).toContain(`"profile":"${profile}","samples":25`); + } + expect(workflow).toContain( + '--html-input "$corpus_dir/${{ matrix.profile }}.html"', + ); + expect(workflow).toContain( + '--revision-input "$corpus_dir/${{ matrix.profile }}.envelope.json"', + ); + expect(workflow).toContain('node benchmarks/run-current-suite.mjs'); + expect(workflow).toContain( + '--resulting-input "$corpus_dir/${{ matrix.profile }}.changed.envelope.json"', + ); + expect(workflow).toContain('--profile "${{ matrix.profile }}"'); + expect(workflow).toContain('--samples "${{ matrix.samples }}"'); + expect(workflow).toContain( + '--reference-hardware-id github-actions-ubuntu-24.04-x64', + ); + expect(workflow).toContain( + 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a', + ); + expect(workflow).toContain('retention-days: 5'); + expect(workflow).not.toContain('secrets.'); + expect(workflow).not.toContain('contents: write'); + expect(workflow).not.toContain('pull-requests: write'); + }); +}); diff --git a/test/setup.ts b/test/setup.ts index 5bf253a35..4ecf96d76 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -1,4 +1,11 @@ import '@testing-library/jest-dom/vitest'; +import { realpathSync } from 'node:fs'; +import { tmpdir } from 'node:os'; + +// macOS exposes its system temp directory through /var -> /private/var. Give +// child-process path guards the canonical platform path so they can still +// reject symlinks created inside the test boundary. +process.env.TMPDIR = realpathSync(tmpdir()); // jsdom does not implement canvas; the downscale path falls back gracefully, // but stubbing getContext keeps any accidental calls from throwing. diff --git a/vite.demo.chunking.ts b/vite.demo.chunking.ts new file mode 100644 index 000000000..87cc28346 --- /dev/null +++ b/vite.demo.chunking.ts @@ -0,0 +1,54 @@ +const NODE_MODULES = '/node_modules/'; + +function hasPackage(moduleId: string, packageName: string): boolean { + return moduleId.includes(`${NODE_MODULES}${packageName}/`); +} + +/** + * Assign large demo-only dependency families to stable Rollup vendor chunks. + * Product/library entry points are unchanged; this only keeps the standalone + * buyer demo from regressing into one oversized JavaScript payload. + */ +export function demoVendorChunk(id: string): string | undefined { + const moduleId = id.replace(/\\/g, '/'); + + if (!moduleId.includes(NODE_MODULES)) { + return undefined; + } + + if ( + hasPackage(moduleId, 'react') || + hasPackage(moduleId, 'react-dom') || + hasPackage(moduleId, 'scheduler') + ) { + return 'react-vendor'; + } + + if ( + hasPackage(moduleId, '@tiptap/pm') || + moduleId.includes(`${NODE_MODULES}prosemirror-`) + ) { + return 'prosemirror-vendor'; + } + + if (moduleId.includes(`${NODE_MODULES}@tiptap/`)) { + return 'tiptap-vendor'; + } + + if ( + hasPackage(moduleId, 'marked') || + hasPackage(moduleId, 'turndown') || + hasPackage(moduleId, 'turndown-plugin-gfm') + ) { + return 'serialization-vendor'; + } + + if ( + hasPackage(moduleId, 'yjs') || + hasPackage(moduleId, 'y-prosemirror') + ) { + return 'collaboration-vendor'; + } + + return 'vendor'; +} diff --git a/vite.demo.config.ts b/vite.demo.config.ts index dd5e2d187..e5d2a0b6f 100644 --- a/vite.demo.config.ts +++ b/vite.demo.config.ts @@ -1,6 +1,7 @@ import { resolve } from 'node:path'; import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; +import { demoVendorChunk } from './vite.demo.chunking'; // Standalone demo build (Vite). `pnpm build:demo` emits a static site to // dist-demo/ that can be served by any static host or the provided Dockerfile. @@ -11,5 +12,10 @@ export default defineConfig({ build: { outDir: resolve(__dirname, 'dist-demo'), emptyOutDir: true, + rollupOptions: { + output: { + manualChunks: demoVendorChunk, + }, + }, }, });