diff --git a/.jules/bolt.md b/.jules/bolt.md index b08b203a..86388713 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -4,3 +4,6 @@ ## 2026-07-12 - Optimize renderTaskRow DOM allocations **Learning:** Caching unattached template nodes and instantiating them via `.cloneNode(false)` reduces DOM instantiation overhead in O(N) render loops significantly. **Action:** Apply this optimization to other hot-path rendering elements such as rows, cells, and stack containers. +## 2026-08-25 - Benchmark fixed-width date formatting before accepting micro-optimizations +**Learning:** Engine-internal explanations are not acceptance evidence. A fixed-width date formatter optimization must preserve every observable output while demonstrating a material median improvement against an immutable protected-base revision in the same browser/runtime. +**Action:** Keep date-format micro-optimizations only when the protected Chromium A/B benchmark proves exact semantic parity across a leap-year date corpus and at least a 10% median improvement; otherwise prefer the clearer implementation. diff --git a/app.js b/app.js index a04aae71..a1e722fd 100644 --- a/app.js +++ b/app.js @@ -2682,22 +2682,32 @@ function clamp(value, min, max) { return Math.min(max, Math.max(min, value)); } +// ⚡ Bolt Optimization: Use inline ternary operator instead of String.padStart() +// to avoid unnecessary string object allocations and JS-to-C++ overhead in hot loops. function formatDateInput(date) { const year = date.getUTCFullYear(); - const month = String(date.getUTCMonth() + 1).padStart(2, '0'); - const day = String(date.getUTCDate()).padStart(2, '0'); + const m = date.getUTCMonth() + 1; + const d = date.getUTCDate(); + const month = m < 10 ? '0' + m : '' + m; + const day = d < 10 ? '0' + d : '' + d; return `${year}-${month}-${day}`; } function formatLocalDateInput(date) { const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, '0'); - const day = String(date.getDate()).padStart(2, '0'); + const m = date.getMonth() + 1; + const d = date.getDate(); + const month = m < 10 ? '0' + m : '' + m; + const day = d < 10 ? '0' + d : '' + d; return `${year}-${month}-${day}`; } function formatCompactDate(date) { - return `${date.getFullYear()}${String(date.getMonth() + 1).padStart(2, '0')}${String(date.getDate()).padStart(2, '0')}`; + const m = date.getMonth() + 1; + const d = date.getDate(); + const month = m < 10 ? '0' + m : '' + m; + const day = d < 10 ? '0' + d : '' + d; + return `${date.getFullYear()}${month}${day}`; } function formatPercent(value, digits) { diff --git a/package.json b/package.json index 8cefdc74..6137b606 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,12 @@ "coverage": "npm run test:coverage", "server": "node server/server.mjs", "test:api": "node tests/api/auth-secret.test.mjs && node tests/api/smoke.mjs && node tests/api/ratelimit.test.mjs && node tests/api/attachment-status.test.mjs && node tests/api/session-revocation.test.mjs && node tests/api/orchestrator-attribution.test.mjs", - "test:unit": "node tests/unit/opencode-config.test.mjs && node tests/unit/changelog-release-notes.test.mjs && node tests/unit/analytics.test.mjs && node tests/unit/cpm.test.mjs && node tests/unit/baseline-compare.test.mjs && node tests/unit/workload.test.mjs && node tests/unit/cost-evm.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && node tests/unit/dep-types.test.mjs && node tests/unit/weekly-report.test.mjs && node tests/unit/clearfolio.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/sprint-stats.test.mjs && node tests/unit/burndown.test.mjs && node tests/unit/pm-analysis.test.mjs && node tests/unit/cloud-sync-security.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/coverage-script-contract.test.mjs && node tests/unit/toast-accessibility.test.mjs", + "test:unit": "node tests/unit/opencode-config.test.mjs && node tests/unit/changelog-release-notes.test.mjs && node tests/unit/analytics.test.mjs && node tests/unit/cpm.test.mjs && node tests/unit/baseline-compare.test.mjs && node tests/unit/workload.test.mjs && node tests/unit/cost-evm.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && node tests/unit/dep-types.test.mjs && node tests/unit/weekly-report.test.mjs && node tests/unit/clearfolio.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/sprint-stats.test.mjs && node tests/unit/burndown.test.mjs && node tests/unit/pm-analysis.test.mjs && node tests/unit/cloud-sync-security.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/coverage-script-contract.test.mjs && node tests/unit/toast-accessibility.test.mjs && node tests/unit/date-format-benchmark-order.test.mjs", "test:coverage": "c8 --all --include=app.js --include=cloud-sync.js --include=scripts/ci/static_coverage_evidence.mjs --include=server/attachment_status.mjs --include=server/app.mjs --include=server/auth.mjs --include=server/clearfolio.mjs --include=server/orchestrator.mjs --reporter=json --reporter=json-summary npm run test:coverage:cases", "test:coverage:cases": "node tests/unit/coverage-script-contract.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && npm run test:api", - "test:e2e": "playwright test", - "test:e2e:headed": "playwright test --headed", - "test:e2e:cloud": "playwright install chromium && playwright test tests/e2e/cloud.spec.js tests/e2e/toast-accessibility.spec.js", + "test:e2e": "playwright test --grep-invert @benchmark", + "test:e2e:headed": "playwright test --headed --grep-invert @benchmark", + "test:e2e:cloud": "playwright install chromium && playwright test tests/e2e/cloud.spec.js tests/e2e/toast-accessibility.spec.js tests/e2e/date-format-performance.spec.js", "test:fuzz": "playwright install chromium && playwright test tests/e2e/csv_formula_fuzz.spec.js", "fuzz": "node --test tests/fuzz/*.mjs" }, diff --git a/tests/e2e/date-format-performance.spec.js b/tests/e2e/date-format-performance.spec.js new file mode 100644 index 00000000..aea9578e --- /dev/null +++ b/tests/e2e/date-format-performance.spec.js @@ -0,0 +1,360 @@ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +import { test, expect } from '@playwright/test'; + +import { + counterbalancedBenchmarkRounds, + stableBenchmarkChecksum, + summarizeCounterbalancedMeasurements, +} from '../helpers/date-format-benchmark.mjs'; + +test.describe.configure({ retries: process.env.CI ? 2 : 0 }); + +const ITERATION_COUNT = 400_000; +const SAMPLE_COUNT = 7; +const WARMUP_COUNT = 3; +const TARGET_IMPROVEMENT_PERCENT = 10; +const DEFAULT_BENCHMARK_BASE_REF = 'develop'; + +function assertImmutableSha(candidate, label) { + const sha = String(candidate || '').trim(); + if (!/^[a-f0-9]{40}$/.test(sha) || /^0+$/.test(sha)) { + throw new Error(`Missing immutable ${label} SHA: ${sha || ''}`); + } + return sha; +} + +function assertBenchmarkBaseRef(candidate) { + const baseRef = String(candidate || '').trim(); + if (!baseRef || baseRef.length > 255 || /[\u0000-\u001f\u007f]/u.test(baseRef)) { + throw new Error(`Missing or invalid benchmark base ref: ${baseRef || ''}`); + } + return baseRef; +} + +function githubEvent() { + const eventPath = process.env.GITHUB_EVENT_PATH; + return eventPath ? JSON.parse(readFileSync(eventPath, 'utf8')) : {}; +} + +function readOriginBranchTip(baseRef) { + const branch = assertBenchmarkBaseRef(baseRef); + const fullRef = `refs/heads/${branch}`; + let output; + try { + output = execFileSync('git', ['ls-remote', '--heads', 'origin', fullRef], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + throw new Error(`Unable to resolve live benchmark base ${fullRef}`); + } + + const matches = output + .split(/\r?\n/u) + .filter(Boolean) + .map((line) => line.split(/\s+/u)) + .filter(([, remoteRef]) => remoteRef === fullRef); + if (matches.length !== 1) { + throw new Error(`Expected exactly one live benchmark base for ${fullRef}, found ${matches.length}`); + } + return assertImmutableSha(matches[0][0], `live benchmark base ${fullRef}`); +} + +function readCurrentHeadSha() { + let output; + try { + output = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + throw new Error('Unable to resolve local benchmark contributor HEAD'); + } + return assertImmutableSha(output, 'local benchmark contributor head'); +} + +function resolveBenchmarkBaseSha(event, readLiveBaseSha = readOriginBranchTip) { + const override = String(process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA || '').trim(); + const pullRequestBase = event?.pull_request?.base; + if (!pullRequestBase) { + const eventBaseSha = String(event?.before || '').trim(); + if (override) { + return assertImmutableSha(override, 'benchmark base'); + } + if (eventBaseSha && !/^0{40}$/u.test(eventBaseSha)) { + return assertImmutableSha(eventBaseSha, 'benchmark base'); + } + const localBaseRef = assertBenchmarkBaseRef( + process.env.SCOPEWEAVE_BENCHMARK_BASE_REF || DEFAULT_BENCHMARK_BASE_REF, + ); + return assertImmutableSha( + readLiveBaseSha(localBaseRef), + `live benchmark base ${localBaseRef}`, + ); + } + + const eventBaseSha = assertImmutableSha(pullRequestBase.sha, 'benchmark base'); + const baseRef = assertBenchmarkBaseRef(pullRequestBase.ref); + const liveBaseSha = assertImmutableSha( + readLiveBaseSha(baseRef), + `live benchmark base ${baseRef}`, + ); + if (liveBaseSha !== eventBaseSha) { + throw new Error( + `Protected base moved from ${eventBaseSha} to ${liveBaseSha}; regenerate benchmark against fresh ${baseRef}`, + ); + } + if (override) { + const overrideSha = assertImmutableSha(override, 'benchmark base override'); + if (overrideSha !== liveBaseSha) { + throw new Error( + `Benchmark base override ${overrideSha} does not match live protected base ${liveBaseSha}`, + ); + } + } + return liveBaseSha; +} + +test('pull request benchmark refuses a stale protected-base event snapshot', () => { + const originalOverride = process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + try { + const eventBaseSha = 'a'.repeat(40); + const liveBaseSha = 'b'.repeat(40); + const event = { + pull_request: { + base: { sha: eventBaseSha, ref: 'develop' }, + }, + }; + + expect(() => resolveBenchmarkBaseSha(event, () => liveBaseSha)).toThrow(/protected base moved/i); + } finally { + if (originalOverride === undefined) { + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + } else { + process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA = originalOverride; + } + } +}); + +test('first-push zero before SHA falls back to the live protected base', () => { + const originalOverride = process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + const originalBaseRef = process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + try { + const liveBaseSha = 'c'.repeat(40); + expect(resolveBenchmarkBaseSha({ before: '0'.repeat(40) }, (baseRef) => { + expect(baseRef).toBe(DEFAULT_BENCHMARK_BASE_REF); + return liveBaseSha; + })).toBe(liveBaseSha); + } finally { + if (originalOverride === undefined) delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + else process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA = originalOverride; + if (originalBaseRef === undefined) delete process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + else process.env.SCOPEWEAVE_BENCHMARK_BASE_REF = originalBaseRef; + } +}); + +function resolveBenchmarkCandidateSha(event, readLocalHeadSha = readCurrentHeadSha) { + const override = String(process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA || '').trim(); + if (override) return assertImmutableSha(override, 'benchmark contributor head'); + const eventCandidateSha = event?.pull_request?.head?.sha || event?.after || process.env.GITHUB_SHA; + if (eventCandidateSha) { + return assertImmutableSha(eventCandidateSha, 'benchmark contributor head'); + } + return assertImmutableSha(readLocalHeadSha(), 'local benchmark contributor head'); +} + +test('documented cloud benchmark resolves local-clone revisions without a GitHub event', () => { + const originalBaseOverride = process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + const originalBaseRef = process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + const originalHeadOverride = process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA; + const originalGitHubSha = process.env.GITHUB_SHA; + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + delete process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + delete process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA; + delete process.env.GITHUB_SHA; + try { + const liveBaseSha = 'c'.repeat(40); + const localHeadSha = 'd'.repeat(40); + expect(resolveBenchmarkBaseSha({}, (baseRef) => { + expect(baseRef).toBe(DEFAULT_BENCHMARK_BASE_REF); + return liveBaseSha; + })).toBe(liveBaseSha); + expect(resolveBenchmarkCandidateSha({}, () => localHeadSha)).toBe(localHeadSha); + } finally { + if (originalBaseOverride === undefined) delete process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA; + else process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA = originalBaseOverride; + if (originalBaseRef === undefined) delete process.env.SCOPEWEAVE_BENCHMARK_BASE_REF; + else process.env.SCOPEWEAVE_BENCHMARK_BASE_REF = originalBaseRef; + if (originalHeadOverride === undefined) delete process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA; + else process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA = originalHeadOverride; + if (originalGitHubSha === undefined) delete process.env.GITHUB_SHA; + else process.env.GITHUB_SHA = originalGitHubSha; + } +}); + +function readGitFile(commitSha, path) { + const spec = `${commitSha}:${path}`; + try { + return execFileSync('git', ['show', spec], { encoding: 'utf8' }); + } catch { + execFileSync('git', ['fetch', '--depth=1', 'origin', commitSha], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + return execFileSync('git', ['show', spec], { encoding: 'utf8' }); + } +} + +function instrumentDateSource(source) { + const bootstrapCall = '\nbootstrap();'; + const bootstrapIndex = source.lastIndexOf(bootstrapCall); + if (bootstrapIndex === -1) { + throw new Error('Benchmark app source is missing the expected bootstrap call'); + } + + const withoutBootstrap = `${source.slice(0, bootstrapIndex)}${source.slice(bootstrapIndex + bootstrapCall.length)}`; + return `${withoutBootstrap}\n\nwindow.__scopeweaveDateBenchmark = Object.freeze({\n formatDateInput,\n formatLocalDateInput,\n formatCompactDate,\n});\n`; +} + +async function measureDateFormatting(browser, { appSource, label }) { + const context = await browser.newContext(); + try { + const page = await context.newPage(); + await page.route('**/app.js', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/javascript; charset=utf-8', + body: instrumentDateSource(appSource), + }); + }); + + await page.goto('/'); + const result = await page.evaluate(({ iterationCount, sampleCount, warmupCount }) => { + const benchmark = window.__scopeweaveDateBenchmark; + if (!benchmark) throw new Error('date-format benchmark bridge did not initialize'); + + const dates = Array.from( + { length: 366 }, + (_, index) => new Date(Date.UTC(2024, 0, index + 1, 12, 0, 0)), + ); + + const run = () => { + let checksum = 0; + for (let index = 0; index < iterationCount; index += 1) { + const date = dates[index % dates.length]; + const utc = benchmark.formatDateInput(date); + const local = benchmark.formatLocalDateInput(date); + const compact = benchmark.formatCompactDate(date); + checksum = ( + checksum + + utc.charCodeAt(5) + + utc.charCodeAt(8) + + local.charCodeAt(5) + + local.charCodeAt(8) + + compact.charCodeAt(4) + + compact.charCodeAt(7) + ) >>> 0; + } + return checksum; + }; + + for (let warmup = 0; warmup < warmupCount; warmup += 1) { + run(); + } + + const samples = []; + const sampleChecksums = []; + for (let sample = 0; sample < sampleCount; sample += 1) { + const startedAt = performance.now(); + sampleChecksums.push(run()); + samples.push(performance.now() - startedAt); + } + + const semanticSnapshot = JSON.stringify(dates.map((date) => [ + benchmark.formatDateInput(date), + benchmark.formatLocalDateInput(date), + benchmark.formatCompactDate(date), + ])); + + return { samples, sampleChecksums, semanticSnapshot }; + }, { + iterationCount: ITERATION_COUNT, + sampleCount: SAMPLE_COUNT, + warmupCount: WARMUP_COUNT, + }); + + return { + label, + samples: result.samples, + checksum: stableBenchmarkChecksum(result.sampleChecksums), + semanticSnapshot: result.semanticSnapshot, + }; + } finally { + await context.close(); + } +} + +test('@benchmark fixed-width date formatting preserves exact semantics and beats the protected base', async ({ browser }) => { + test.setTimeout(240_000); + + const event = githubEvent(); + const baseSha = resolveBenchmarkBaseSha(event); + const candidateSha = resolveBenchmarkCandidateSha(event); + const sourceByLabel = new Map([ + ['protected-base', readGitFile(baseSha, 'app.js')], + ['exact-contributor-head', readGitFile(candidateSha, 'app.js')], + ]); + const measurementOrder = counterbalancedBenchmarkRounds(); + const measurements = []; + + for (const round of measurementOrder) { + for (const label of round) { + measurements.push(await measureDateFormatting(browser, { + appSource: sourceByLabel.get(label), + label, + })); + } + } + + for (const measurement of measurements) { + expect(measurement.samples).toHaveLength(SAMPLE_COUNT); + expect(measurement.samples.every((duration) => duration > 0)).toBe(true); + } + + const summary = summarizeCounterbalancedMeasurements(measurements); + expect(summary.baselineMedianDurationMs).toBeGreaterThan(0); + expect(summary.candidateMedianDurationMs).toBeGreaterThan(0); + expect( + summary.improvementPercent, + `expected >=${TARGET_IMPROVEMENT_PERCENT}% counterbalanced median date-format improvement for exact head ${candidateSha} over protected base ${baseSha}, got ${summary.improvementPercent.toFixed(2)}%`, + ).toBeGreaterThanOrEqual(TARGET_IMPROVEMENT_PERCENT); + + const completionBaseSha = resolveBenchmarkBaseSha(event); + expect(completionBaseSha).toBe(baseSha); + + console.log(`SCOPEWEAVE_DATE_FORMAT_BENCHMARK ${JSON.stringify({ + iterationCount: ITERATION_COUNT, + sampleCountPerMeasurement: SAMPLE_COUNT, + warmupCountPerMeasurement: WARMUP_COUNT, + measurementOrder, + protectedBaseSha: baseSha, + exactContributorHeadSha: candidateSha, + targetImprovementPercent: TARGET_IMPROVEMENT_PERCENT, + improvementPercent: summary.improvementPercent, + baseline: { + samples: summary.baselineSamples, + medianDurationMs: summary.baselineMedianDurationMs, + checksum: summary.checksum, + }, + optimized: { + samples: summary.candidateSamples, + medianDurationMs: summary.candidateMedianDurationMs, + checksum: summary.checksum, + }, + })}`); +}); diff --git a/tests/helpers/date-format-benchmark.mjs b/tests/helpers/date-format-benchmark.mjs new file mode 100644 index 00000000..ab8e3820 --- /dev/null +++ b/tests/helpers/date-format-benchmark.mjs @@ -0,0 +1,111 @@ +const BASELINE_LABEL = 'protected-base'; +const CANDIDATE_LABEL = 'exact-contributor-head'; + +/** + * Return the two benchmark rounds needed to neutralize execution-position bias. + * + * Each revision runs once first and once second. This prevents a systematic + * warm-cache, JIT, thermal, or browser-process advantage from being attributed + * to whichever revision happens to run second every time. + * + * @returns {ReadonlyArray>} Counterbalanced revision labels. + */ +export function counterbalancedBenchmarkRounds() { + return Object.freeze([ + Object.freeze([BASELINE_LABEL, CANDIDATE_LABEL]), + Object.freeze([CANDIDATE_LABEL, BASELINE_LABEL]), + ]); +} + +/** + * Return one semantic checksum only when every timed run produced the same value. + * + * The benchmark must not combine checksums with XOR because an even sample count + * would cancel identical values to zero and silently destroy the evidence. This + * helper instead keeps the first value and fails closed if any later timed run + * produces different formatting output. + * + * @param {number[]} values Unsigned integer checksums from individual timed runs. + * @returns {number} The stable checksum shared by every timed run. + */ +export function stableBenchmarkChecksum(values) { + if (!Array.isArray(values) || values.length === 0) { + throw new Error('benchmark checksum samples must be a non-empty array'); + } + const reference = values[0]; + if (!Number.isSafeInteger(reference) || reference < 0) { + throw new Error('benchmark checksum must be a non-negative safe integer'); + } + for (const value of values) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error('benchmark checksum must be a non-negative safe integer'); + } + if (value !== reference) { + throw new Error('date-format checksum changed between samples'); + } + } + return reference; +} + +function median(values) { + if (!Array.isArray(values) || values.length === 0) { + throw new Error('benchmark samples must be a non-empty array'); + } + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[midpoint]; + return (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +/** + * Aggregate two counterbalanced A/B rounds without losing semantic evidence. + * + * @param {Array<{label: string, samples: number[], semanticSnapshot: string, checksum: number}>} measurements + * Measurements emitted in the execution order declared by + * {@link counterbalancedBenchmarkRounds}. + * @returns {{baselineSamples: number[], candidateSamples: number[], baselineMedianDurationMs: number, candidateMedianDurationMs: number, improvementPercent: number, semanticSnapshot: string, checksum: number}} + * Combined timing and semantic evidence for the protected base and candidate. + */ +export function summarizeCounterbalancedMeasurements(measurements) { + if (!Array.isArray(measurements) || measurements.length !== 4) { + throw new Error('counterbalanced benchmark requires exactly four measurements'); + } + + const baselineMeasurements = measurements.filter(({ label }) => label === BASELINE_LABEL); + const candidateMeasurements = measurements.filter(({ label }) => label === CANDIDATE_LABEL); + if (baselineMeasurements.length !== 2 || candidateMeasurements.length !== 2) { + throw new Error('counterbalanced benchmark requires two measurements per revision'); + } + + const reference = measurements[0]; + for (const measurement of measurements) { + if (!Array.isArray(measurement.samples) || measurement.samples.length === 0) { + throw new Error(`benchmark measurement ${measurement.label} has no samples`); + } + if (measurement.semanticSnapshot !== reference.semanticSnapshot) { + throw new Error('date-format semantic snapshots differ across benchmark rounds'); + } + if (measurement.checksum !== reference.checksum) { + throw new Error('date-format checksums differ across benchmark rounds'); + } + } + + const baselineSamples = baselineMeasurements.flatMap(({ samples }) => samples); + const candidateSamples = candidateMeasurements.flatMap(({ samples }) => samples); + const baselineMedianDurationMs = median(baselineSamples); + const candidateMedianDurationMs = median(candidateSamples); + const improvementPercent = ( + (baselineMedianDurationMs - candidateMedianDurationMs) + / baselineMedianDurationMs + ) * 100; + + return Object.freeze({ + baselineSamples, + candidateSamples, + baselineMedianDurationMs, + candidateMedianDurationMs, + improvementPercent, + semanticSnapshot: reference.semanticSnapshot, + checksum: reference.checksum, + }); +} diff --git a/tests/unit/date-format-benchmark-order.test.mjs b/tests/unit/date-format-benchmark-order.test.mjs new file mode 100644 index 00000000..85048139 --- /dev/null +++ b/tests/unit/date-format-benchmark-order.test.mjs @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +import { + counterbalancedBenchmarkRounds, + stableBenchmarkChecksum, + summarizeCounterbalancedMeasurements, +} from '../helpers/date-format-benchmark.mjs'; + +const rounds = counterbalancedBenchmarkRounds(); +assert.deepEqual( + rounds, + [ + ['protected-base', 'exact-contributor-head'], + ['exact-contributor-head', 'protected-base'], + ], + 'the benchmark must measure each revision once in each execution position', +); + +assert.equal( + stableBenchmarkChecksum([42, 42, 42, 42]), + 42, + 'an even number of stable samples must preserve the semantic checksum instead of cancelling to zero', +); +assert.throws( + () => stableBenchmarkChecksum([42, 42, 43, 42]), + /checksum changed between samples/i, + 'the benchmark must fail closed if one timed run produces different semantic evidence', +); + +const packageJson = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')); +const benchmarkSpec = readFileSync( + new URL('../e2e/date-format-performance.spec.js', import.meta.url), + 'utf8', +); +assert.match( + packageJson.scripts['test:e2e'], + /--grep-invert\s+["']?@benchmark["']?/u, + 'the ordinary offline e2e command must exclude network-dependent benchmark tests', +); +assert.match( + benchmarkSpec, + /test\(['"]@benchmark\b/u, + 'the network-dependent performance acceptance test must carry the benchmark marker excluded by ordinary e2e runs', +); +assert.match( + packageJson.scripts['test:e2e:cloud'], + /date-format-performance\.spec\.js/u, + 'the protected cloud e2e path must continue to run the benchmark explicitly', +); + +const syntheticSecondRunAdvantage = [ + { + label: 'protected-base', + samples: Array(7).fill(10), + semanticSnapshot: 'same-output', + checksum: 42, + }, + { + label: 'exact-contributor-head', + samples: Array(7).fill(8), + semanticSnapshot: 'same-output', + checksum: 42, + }, + { + label: 'exact-contributor-head', + samples: Array(7).fill(10), + semanticSnapshot: 'same-output', + checksum: 42, + }, + { + label: 'protected-base', + samples: Array(7).fill(8), + semanticSnapshot: 'same-output', + checksum: 42, + }, +]; + +const summary = summarizeCounterbalancedMeasurements(syntheticSecondRunAdvantage); +assert.equal(summary.baselineMedianDurationMs, 9); +assert.equal(summary.candidateMedianDurationMs, 9); +assert.equal( + summary.improvementPercent, + 0, + 'counterbalancing must neutralize an execution-position speedup instead of misattributing it to the candidate', +); +assert.deepEqual(summary.baselineSamples, [ + ...Array(7).fill(10), + ...Array(7).fill(8), +]); +assert.deepEqual(summary.candidateSamples, [ + ...Array(7).fill(8), + ...Array(7).fill(10), +]); + +console.log('✓ date-format benchmark execution-order regression passed');