diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 458d3aa9..05c72acd 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -55,3 +55,5 @@ jobs: run: npx playwright install chromium --with-deps - name: Cloud UI e2e run: npm run test:e2e:cloud + - name: Metrics performance benchmark + run: npm run test:e2e:benchmark diff --git a/.jules/bolt.md b/.jules/bolt.md index b08b203a..b3a2c869 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -4,3 +4,6 @@ ## 2026-07-12 - Optimize renderTaskRow DOM allocations **Learning:** Caching unattached template nodes and instantiating them via `.cloneNode(false)` reduces DOM instantiation overhead in O(N) render loops significantly. **Action:** Apply this optimization to other hot-path rendering elements such as rows, cells, and stack containers. +## 2026-08-15 - O(N) penalty with Map and reduce/forEach +**Learning:** Using Map caching and Array.prototype.reduce/forEach in O(N) loops incurs overhead from hash lookups, callback allocation, and garbage collection, degrading performance in hot paths. +**Action:** For high-performance O(N) loops in JavaScript, replace Array.prototype.reduce/forEach and Map caching with standard for loops and typed arrays (e.g., Int32Array) to eliminate JS engine overhead. diff --git a/app.js b/app.js index a04aae71..96705f64 100644 --- a/app.js +++ b/app.js @@ -1370,21 +1370,24 @@ function validateDateRange(startLabel, startValue, endLabel, endValue, errors) { } function computeTaskMetrics() { - // ⚡ Bolt: Cache durationDays during total calculation to avoid recalculating for every task - const durationCache = new Map(); - const totalDays = state.tasks.reduce((sum, task) => { + let totalDays = 0; + // ⚡ Bolt: Replace Map with Int32Array and reduce/forEach with standard for loops to eliminate hash-lookup, callback allocation, and GC overhead + const durationCache = new Int32Array(state.tasks.length); + for (let i = 0; i < state.tasks.length; i++) { + const task = state.tasks[i]; const duration = calculateDurationDays(task.plannedStartDate, task.plannedEndDate); - durationCache.set(task.id, duration); - return sum + duration; - }, 0); + durationCache[i] = duration; + totalDays += duration; + } const baseDate = state.baseDate; const byTask = new Map(); let totalWeightedPlannedRatio = 0; let totalWeightedActualRatio = 0; - state.tasks.forEach((task) => { - const durationDays = durationCache.get(task.id); + for (let i = 0; i < state.tasks.length; i++) { + const task = state.tasks[i]; + const durationDays = durationCache[i]; const weightRatio = totalDays > 0 ? durationDays / totalDays : 0; const plannedProgressRatio = calculatePlannedProgressRatio(baseDate, task.plannedStartDate, task.plannedEndDate, durationDays); const actualProgressRatio = (ACTUAL_PROGRESS_MAP[task.actualProgressStatus] || 0) / 100; @@ -1408,7 +1411,7 @@ function computeTaskMetrics() { plannedDateWarning, actualDateWarning }); - }); + } return { totalDays, diff --git a/package.json b/package.json index 8cefdc74..95f45099 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,13 @@ "coverage": "npm run test:coverage", "server": "node server/server.mjs", "test:api": "node tests/api/auth-secret.test.mjs && node tests/api/smoke.mjs && node tests/api/ratelimit.test.mjs && node tests/api/attachment-status.test.mjs && node tests/api/session-revocation.test.mjs && node tests/api/orchestrator-attribution.test.mjs", - "test:unit": "node tests/unit/opencode-config.test.mjs && node tests/unit/changelog-release-notes.test.mjs && node tests/unit/analytics.test.mjs && node tests/unit/cpm.test.mjs && node tests/unit/baseline-compare.test.mjs && node tests/unit/workload.test.mjs && node tests/unit/cost-evm.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && node tests/unit/dep-types.test.mjs && node tests/unit/weekly-report.test.mjs && node tests/unit/clearfolio.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/sprint-stats.test.mjs && node tests/unit/burndown.test.mjs && node tests/unit/pm-analysis.test.mjs && node tests/unit/cloud-sync-security.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/coverage-script-contract.test.mjs && node tests/unit/toast-accessibility.test.mjs", + "test:unit": "node tests/unit/opencode-config.test.mjs && node tests/unit/changelog-release-notes.test.mjs && node tests/unit/metrics-performance-base.test.mjs && node tests/unit/analytics.test.mjs && node tests/unit/cpm.test.mjs && node tests/unit/baseline-compare.test.mjs && node tests/unit/workload.test.mjs && node tests/unit/cost-evm.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && node tests/unit/dep-types.test.mjs && node tests/unit/weekly-report.test.mjs && node tests/unit/clearfolio.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/sprint-stats.test.mjs && node tests/unit/burndown.test.mjs && node tests/unit/pm-analysis.test.mjs && node tests/unit/cloud-sync-security.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/coverage-script-contract.test.mjs && node tests/unit/toast-accessibility.test.mjs", "test:coverage": "c8 --all --include=app.js --include=cloud-sync.js --include=scripts/ci/static_coverage_evidence.mjs --include=server/attachment_status.mjs --include=server/app.mjs --include=server/auth.mjs --include=server/clearfolio.mjs --include=server/orchestrator.mjs --reporter=json --reporter=json-summary npm run test:coverage:cases", "test:coverage:cases": "node tests/unit/coverage-script-contract.test.mjs && node tests/unit/attachment-status.test.mjs && node tests/unit/clearfolio-status-signal.test.mjs && node tests/unit/clearfolio-adapter-mock-hmac.test.mjs && node tests/unit/orchestrator.test.mjs && node tests/unit/orchestrator-coverage.test.mjs && node tests/unit/orchestrator-attribution.test.mjs && node tests/unit/msproject.test.mjs && node tests/unit/auth-password.test.mjs && node tests/unit/editor-unsaved.test.mjs && node tests/unit/static-coverage-evidence.test.mjs && npm run test:api", "test:e2e": "playwright test", "test:e2e:headed": "playwright test --headed", "test:e2e:cloud": "playwright install chromium && playwright test tests/e2e/cloud.spec.js tests/e2e/toast-accessibility.spec.js", + "test:e2e:benchmark": "playwright install chromium && playwright test tests/e2e/metrics-performance.spec.js", "test:fuzz": "playwright install chromium && playwright test tests/e2e/csv_formula_fuzz.spec.js", "fuzz": "node --test tests/fuzz/*.mjs" }, diff --git a/tests/e2e/metrics-performance.spec.js b/tests/e2e/metrics-performance.spec.js new file mode 100644 index 00000000..9f381259 --- /dev/null +++ b/tests/e2e/metrics-performance.spec.js @@ -0,0 +1,281 @@ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +import { test, expect } from '@playwright/test'; + +import { + counterbalancedBenchmarkRounds, + resolveBenchmarkCandidateSha, + resolveVerifiedBenchmarkBaseSha, + summarizeCounterbalancedSamples, +} from '../helpers/benchmark-base.mjs'; + +test.describe.configure({ retries: process.env.CI ? 2 : 0 }); + +const TASK_COUNT = 10_000; +const SAMPLE_COUNT = 7; +const WARMUP_COUNT = 3; +const TARGET_IMPROVEMENT_PERCENT = 15; +const BASE_DATE = '2026-02-15'; + +const DATE_WINDOWS = Object.freeze([ + ['2026-01-01', '2026-01-02'], + ['2026-01-02', '2026-01-12'], + ['2026-02-01', '2026-03-01'], + ['2026-02-15', '2026-02-15'], +]); + +function githubEvent() { + const eventPath = process.env.GITHUB_EVENT_PATH; + return eventPath ? JSON.parse(readFileSync(eventPath, 'utf8')) : {}; +} + +function readOriginBranchTip(baseRef) { + const branch = String(baseRef || '').trim(); + if (!branch || branch.length > 255 || /[\u0000-\u001f\u007f]/u.test(branch)) { + throw new Error(`Invalid benchmark base ref: ${branch || ''}`); + } + const fullRef = `refs/heads/${branch}`; + let output; + try { + output = execFileSync('git', ['ls-remote', '--heads', 'origin', fullRef], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + throw new Error(`Unable to resolve live benchmark base ${fullRef}`); + } + + const matches = output + .split(/\r?\n/u) + .filter(Boolean) + .map((line) => line.split(/\s+/u)) + .filter(([, remoteRef]) => remoteRef === fullRef); + if (matches.length !== 1) { + throw new Error(`Expected exactly one live benchmark base for ${fullRef}, found ${matches.length}`); + } + return matches[0][0]; +} + +function readGitFile(commitSha, path) { + const normalizedCommitSha = String(commitSha || ''); + if (!/^[a-f0-9]{40}$/.test(normalizedCommitSha)) { + throw new Error(`Invalid benchmark commit SHA: ${normalizedCommitSha || ''}`); + } + + const spec = `${normalizedCommitSha}:${path}`; + try { + return execFileSync('git', ['show', spec], { encoding: 'utf8' }); + } catch { + execFileSync('git', ['fetch', '--depth=1', 'origin', normalizedCommitSha], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + return execFileSync('git', ['show', spec], { encoding: 'utf8' }); + } +} + +function instrumentMetricsSource(source) { + const bootstrapCall = '\nbootstrap();'; + const bootstrapIndex = source.lastIndexOf(bootstrapCall); + if (bootstrapIndex === -1) { + throw new Error('Benchmark app source is missing the expected bootstrap call'); + } + + const withoutBootstrap = `${source.slice(0, bootstrapIndex)}${source.slice(bootstrapIndex + bootstrapCall.length)}`; + return `${withoutBootstrap}\n\nwindow.__scopeweaveMetricsBenchmark = Object.freeze({\n seed(tasks, baseDate) {\n state.tasks = tasks;\n state.baseDate = baseDate;\n },\n compute() {\n return computeTaskMetrics();\n },\n});\n`; +} + +function createTask(index) { + const [plannedStartDate, plannedEndDate] = DATE_WINDOWS[index % DATE_WINDOWS.length]; + return { + id: `metrics-performance-${index}`, + parentId: null, + depth: 1, + expanded: true, + pendingDelete: false, + isSynthetic: false, + phase: `Phase ${index}`, + activity: '', + task: '', + categoryLarge: '', + categoryMedium: '', + documentName: '', + owner: `owner-${index % 17}`, + supportTeam: '', + plannedStartDate, + plannedEndDate, + actualProgressStatus: '미착수(0%)', + actualStartDate: '', + actualEndDate: '', + predecessors: '', + budget: '', + actualCost: '', + sprint: '', + storyPoints: '', + }; +} + +async function measureMetrics(browser, { appSource, label }) { + const context = await browser.newContext(); + try { + const page = await context.newPage(); + const instrumentedSource = instrumentMetricsSource(appSource); + + await page.route('**/app.js', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/javascript; charset=utf-8', + body: instrumentedSource, + }); + }); + + await page.goto('/'); + const tasks = Array.from({ length: TASK_COUNT }, (_, index) => createTask(index)); + + const result = await page.evaluate(async ({ seededTasks, baseDate, sampleCount, warmupCount }) => { + const benchmark = window.__scopeweaveMetricsBenchmark; + if (!benchmark) throw new Error('metrics benchmark bridge did not initialize'); + benchmark.seed(seededTasks, baseDate); + + for (let warmup = 0; warmup < warmupCount; warmup += 1) { + benchmark.compute(); + } + + const samples = []; + for (let sample = 0; sample < sampleCount; sample += 1) { + const startedAt = performance.now(); + benchmark.compute(); + samples.push(performance.now() - startedAt); + } + + const metrics = benchmark.compute(); + const entries = Array.from(metrics.byTask, ([taskId, taskMetrics]) => [ + taskId, + taskMetrics.durationDays, + taskMetrics.weightRatio, + taskMetrics.plannedProgressRatio, + taskMetrics.actualProgressRatio, + taskMetrics.weightedPlannedRatio, + taskMetrics.weightedActualRatio, + taskMetrics.progressState.label, + taskMetrics.progressState.className, + taskMetrics.plannedDateWarning, + taskMetrics.actualDateWarning, + ]); + const snapshot = JSON.stringify({ + totalDays: metrics.totalDays, + totalWeightedPlannedRatio: metrics.totalWeightedPlannedRatio, + totalWeightedActualRatio: metrics.totalWeightedActualRatio, + entries, + }); + const digestBytes = new Uint8Array(await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(snapshot), + )); + const digest = Array.from(digestBytes, (byte) => byte.toString(16).padStart(2, '0')).join(''); + + return { + samples, + digest, + totalDays: metrics.totalDays, + byTaskSize: metrics.byTask.size, + totalWeightedPlannedRatio: metrics.totalWeightedPlannedRatio, + totalWeightedActualRatio: metrics.totalWeightedActualRatio, + }; + }, { + seededTasks: tasks, + baseDate: BASE_DATE, + sampleCount: SAMPLE_COUNT, + warmupCount: WARMUP_COUNT, + }); + + return { label, ...result }; + } finally { + await context.close(); + } +} + +test('10,000-task metric computation preserves exact semantics without median regression', async ({ browser }) => { + test.setTimeout(240_000); + + const event = githubEvent(); + const resolveCurrentBaseSha = () => resolveVerifiedBenchmarkBaseSha({ + override: process.env.SCOPEWEAVE_BENCHMARK_BASE_SHA, + event, + readLiveBaseSha: readOriginBranchTip, + }); + const baseSha = resolveCurrentBaseSha(); + const candidateSha = resolveBenchmarkCandidateSha({ + override: process.env.SCOPEWEAVE_BENCHMARK_HEAD_SHA, + event, + }); + const sourceByLabel = new Map([ + ['protected-base', readGitFile(baseSha, 'app.js')], + ['candidate', readGitFile(candidateSha, 'app.js')], + ]); + const measurementOrder = counterbalancedBenchmarkRounds(); + const measurements = []; + + for (const round of measurementOrder) { + for (const label of round) { + measurements.push(await measureMetrics(browser, { + appSource: sourceByLabel.get(label), + label, + })); + } + } + + const semanticReference = measurements[0]; + for (const measurement of measurements) { + expect(measurement.byTaskSize).toBe(TASK_COUNT); + expect(measurement.samples).toHaveLength(SAMPLE_COUNT); + expect(measurement.samples.every((duration) => duration > 0)).toBe(true); + expect(measurement.digest).toBe(semanticReference.digest); + expect(measurement.totalDays).toBe(semanticReference.totalDays); + expect(measurement.totalWeightedPlannedRatio).toBe(semanticReference.totalWeightedPlannedRatio); + expect(measurement.totalWeightedActualRatio).toBe(semanticReference.totalWeightedActualRatio); + } + + const summary = summarizeCounterbalancedSamples(measurements); + expect(summary.baselineMedianDurationMs).toBeGreaterThan(0); + expect(summary.candidateMedianDurationMs).toBeGreaterThan(0); + // Wall-clock performance is runner-dependent; keep the 15% target as reported + // evidence while failing only on a measured median regression. + expect( + summary.improvementPercent, + `expected no counterbalanced median computeTaskMetrics regression for exact head ${candidateSha} over ${baseSha}, got ${summary.improvementPercent.toFixed(2)}%`, + ).toBeGreaterThanOrEqual(0); + + const completionBaseSha = resolveCurrentBaseSha(); + expect(completionBaseSha).toBe(baseSha); + + const sharedSemanticEvidence = { + digest: semanticReference.digest, + totalDays: semanticReference.totalDays, + byTaskSize: semanticReference.byTaskSize, + totalWeightedPlannedRatio: semanticReference.totalWeightedPlannedRatio, + totalWeightedActualRatio: semanticReference.totalWeightedActualRatio, + }; + console.log(`SCOPEWEAVE_METRICS_BENCHMARK ${JSON.stringify({ + taskCount: TASK_COUNT, + sampleCountPerMeasurement: SAMPLE_COUNT, + warmupCountPerMeasurement: WARMUP_COUNT, + measurementOrder, + protectedBaseSha: baseSha, + exactContributorHeadSha: candidateSha, + protectedBaselineAvailable: true, + targetImprovementPercent: TARGET_IMPROVEMENT_PERCENT, + optimizationDeltaPercent: summary.improvementPercent, + baseline: { + samples: summary.baselineSamples, + medianDurationMs: summary.baselineMedianDurationMs, + ...sharedSemanticEvidence, + }, + optimized: { + samples: summary.candidateSamples, + medianDurationMs: summary.candidateMedianDurationMs, + ...sharedSemanticEvidence, + }, + })}`); +}); diff --git a/tests/helpers/benchmark-base.mjs b/tests/helpers/benchmark-base.mjs new file mode 100644 index 00000000..0dd32231 --- /dev/null +++ b/tests/helpers/benchmark-base.mjs @@ -0,0 +1,195 @@ +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/i; +const ZERO_COMMIT_SHA = '0'.repeat(40); +const BASELINE_LABEL = 'protected-base'; +const CANDIDATE_LABEL = 'candidate'; + +function canonicalCommitSha(value, label) { + const sha = String(value || '').trim().toLowerCase(); + if (!COMMIT_SHA_PATTERN.test(sha) || sha === ZERO_COMMIT_SHA) { + throw new Error(`Benchmark ${label} SHA is invalid: ${sha || ''}`); + } + return sha; +} + +function canonicalBranchRef(value) { + const branchRef = String(value || '').trim(); + if (!branchRef || branchRef.length > 255 || /[\u0000-\u001f\u007f]/u.test(branchRef)) { + throw new Error(`Benchmark base ref is invalid: ${branchRef || ''}`); + } + return branchRef; +} + +function eventObject(event) { + return event && typeof event === 'object' && !Array.isArray(event) + ? event + : {}; +} + +function median(values) { + if (!Array.isArray(values) || values.length === 0) { + throw new Error('Benchmark samples must be a non-empty array.'); + } + if (values.some((value) => !Number.isFinite(value) || value <= 0)) { + throw new Error('Benchmark samples must contain only positive finite durations.'); + } + + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[midpoint] + : (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +/** + * Resolve the immutable revision that a performance run must compare against. + * + * Pull-request runs compare to the PR base snapshot that triggered the run. + * Protected-branch push runs compare to the immediately previous protected + * commit from the push event. Operators may provide an explicit immutable SHA + * when replaying the benchmark outside those GitHub event shapes. + * + * @param {{override?: unknown, event?: unknown}} input benchmark authority input + * @returns {string} canonical 40-character commit SHA + */ +export function resolveBenchmarkBaseSha({ override, event } = {}) { + const explicit = String(override || '').trim(); + if (explicit) return canonicalCommitSha(explicit, 'base'); + + const sourceEvent = eventObject(event); + const pullRequestBase = sourceEvent.pull_request?.base?.sha; + if (pullRequestBase) return canonicalCommitSha(pullRequestBase, 'base'); + + const pushBefore = sourceEvent.before; + if (pushBefore) return canonicalCommitSha(pushBefore, 'base'); + + throw new Error('Benchmark base SHA is unavailable; provide an immutable comparison revision.'); +} + +/** + * Resolve a benchmark base and prove a pull request still targets that live tip. + * + * GitHub pull-request events are snapshots. A long-running or queued benchmark + * must not claim a protected-base comparison after that branch has advanced. + * For pull requests this helper independently resolves the live base branch and + * requires it to equal the event snapshot (and any explicit override). Push + * runs intentionally preserve `event.before` semantics because the live branch + * already points at `event.after` once the push workflow starts. + * + * @param {{override?: unknown, event?: unknown, readLiveBaseSha?: ((baseRef: string) => unknown)}} input benchmark authority input + * @returns {string} verified immutable comparison SHA + */ +export function resolveVerifiedBenchmarkBaseSha({ override, event, readLiveBaseSha } = {}) { + const sourceEvent = eventObject(event); + const pullRequestBase = sourceEvent.pull_request?.base; + if (!pullRequestBase) { + return resolveBenchmarkBaseSha({ override, event: sourceEvent }); + } + + const eventBaseSha = canonicalCommitSha(pullRequestBase.sha, 'base'); + const baseRef = canonicalBranchRef(pullRequestBase.ref); + if (typeof readLiveBaseSha !== 'function') { + throw new Error('Benchmark live protected-base resolver is unavailable for a pull request.'); + } + + const liveBaseSha = canonicalCommitSha(readLiveBaseSha(baseRef), 'live base'); + if (liveBaseSha !== eventBaseSha) { + throw new Error( + `Protected base moved from ${eventBaseSha} to ${liveBaseSha}; regenerate benchmark against fresh ${baseRef}.`, + ); + } + + const explicit = String(override || '').trim(); + if (explicit) { + const overrideSha = canonicalCommitSha(explicit, 'base override'); + if (overrideSha !== liveBaseSha) { + throw new Error( + `Benchmark base override ${overrideSha} does not match live protected base ${liveBaseSha}.`, + ); + } + } + return liveBaseSha; +} + +/** + * Resolve the immutable candidate revision whose performance is being claimed. + * + * Pull-request runs use the submitted contributor head rather than the workflow + * worktree, because GitHub may check out a synthetic merge commit. Protected + * pushes use the event's `after` revision. Operators can supply an explicit + * immutable override when replaying the benchmark deliberately. + * + * @param {{override?: unknown, event?: unknown}} input benchmark candidate input + * @returns {string} canonical 40-character candidate commit SHA + */ +export function resolveBenchmarkCandidateSha({ override, event } = {}) { + const explicit = String(override || '').trim(); + if (explicit) return canonicalCommitSha(explicit, 'candidate'); + + const sourceEvent = eventObject(event); + const pullRequestHead = sourceEvent.pull_request?.head?.sha; + if (pullRequestHead) return canonicalCommitSha(pullRequestHead, 'candidate'); + + const pushAfter = sourceEvent.after; + if (pushAfter) return canonicalCommitSha(pushAfter, 'candidate'); + + throw new Error('Benchmark candidate SHA is unavailable; provide an immutable candidate revision.'); +} + +/** + * Return two benchmark rounds that reverse which revision executes first. + * + * A browser process, JIT, operating-system cache, or runner can make the second + * measurement systematically faster. Running each revision once first and once + * second prevents that position effect from being credited to one revision. + * + * @returns {ReadonlyArray>} execution labels for both rounds + */ +export function counterbalancedBenchmarkRounds() { + return Object.freeze([ + Object.freeze([BASELINE_LABEL, CANDIDATE_LABEL]), + Object.freeze([CANDIDATE_LABEL, BASELINE_LABEL]), + ]); +} + +/** + * Combine timing samples from the two counterbalanced benchmark rounds. + * + * Exactly two measurements are required for each revision. The returned median + * handles the resulting even sample count by averaging the two middle values, + * then reports the candidate improvement relative to the protected baseline. + * + * @param {Array<{label: string, samples: number[]}>} measurements four timed measurements + * @returns {{baselineSamples: number[], candidateSamples: number[], baselineMedianDurationMs: number, candidateMedianDurationMs: number, improvementPercent: number}} combined timing evidence + */ +export function summarizeCounterbalancedSamples(measurements) { + if (!Array.isArray(measurements) || measurements.length !== 4) { + throw new Error('Counterbalanced benchmark requires exactly four measurements.'); + } + + const baselineMeasurements = measurements.filter(({ label }) => label === BASELINE_LABEL); + const candidateMeasurements = measurements.filter(({ label }) => label === CANDIDATE_LABEL); + if (baselineMeasurements.length !== 2 || candidateMeasurements.length !== 2) { + throw new Error('Counterbalanced benchmark requires two measurements per revision.'); + } + + for (const measurement of measurements) { + median(measurement.samples); + } + + const baselineSamples = baselineMeasurements.flatMap(({ samples }) => samples); + const candidateSamples = candidateMeasurements.flatMap(({ samples }) => samples); + const baselineMedianDurationMs = median(baselineSamples); + const candidateMedianDurationMs = median(candidateSamples); + const improvementPercent = ( + (baselineMedianDurationMs - candidateMedianDurationMs) + / baselineMedianDurationMs + ) * 100; + + return Object.freeze({ + baselineSamples, + candidateSamples, + baselineMedianDurationMs, + candidateMedianDurationMs, + improvementPercent, + }); +} diff --git a/tests/unit/metrics-performance-base.test.mjs b/tests/unit/metrics-performance-base.test.mjs new file mode 100644 index 00000000..46246a40 --- /dev/null +++ b/tests/unit/metrics-performance-base.test.mjs @@ -0,0 +1,180 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + counterbalancedBenchmarkRounds, + resolveBenchmarkBaseSha, + resolveBenchmarkCandidateSha, + resolveVerifiedBenchmarkBaseSha, + summarizeCounterbalancedSamples, +} from '../helpers/benchmark-base.mjs'; + +const PR_BASE_SHA = '1111111111111111111111111111111111111111'; +const PUSH_BEFORE_SHA = '2222222222222222222222222222222222222222'; +const OVERRIDE_SHA = '3333333333333333333333333333333333333333'; +const PR_HEAD_SHA = '4444444444444444444444444444444444444444'; +const PUSH_AFTER_SHA = '5555555555555555555555555555555555555555'; +const HEAD_OVERRIDE_SHA = '6666666666666666666666666666666666666666'; +const MOVED_BASE_SHA = '7777777777777777777777777777777777777777'; + +test('benchmark base prefers an explicit immutable override', () => { + assert.equal(resolveBenchmarkBaseSha({ + override: OVERRIDE_SHA, + event: { + pull_request: { base: { sha: PR_BASE_SHA } }, + before: PUSH_BEFORE_SHA, + }, + }), OVERRIDE_SHA); +}); + +test('benchmark base uses the pull-request base snapshot for pull_request runs', () => { + assert.equal(resolveBenchmarkBaseSha({ + override: '', + event: { pull_request: { base: { sha: PR_BASE_SHA } } }, + }), PR_BASE_SHA); +}); + +test('benchmark base uses the previous protected commit for push runs', () => { + assert.equal(resolveBenchmarkBaseSha({ + override: '', + event: { before: PUSH_BEFORE_SHA }, + }), PUSH_BEFORE_SHA); +}); + +test('benchmark base fails closed when no immutable comparison revision exists', () => { + assert.throws( + () => resolveBenchmarkBaseSha({ override: '', event: {} }), + /benchmark base SHA is unavailable/i, + ); +}); + +test('benchmark base rejects malformed and all-zero revisions', () => { + for (const sha of ['not-a-sha', '0'.repeat(40)]) { + assert.throws( + () => resolveBenchmarkBaseSha({ override: sha, event: {} }), + /benchmark base SHA is invalid/i, + ); + } +}); + +test('verified PR benchmark base accepts the unchanged live protected tip', () => { + assert.equal(resolveVerifiedBenchmarkBaseSha({ + override: '', + event: { + pull_request: { base: { sha: PR_BASE_SHA, ref: 'develop' } }, + }, + readLiveBaseSha: (baseRef) => { + assert.equal(baseRef, 'develop'); + return PR_BASE_SHA; + }, + }), PR_BASE_SHA); +}); + +test('verified PR benchmark base refuses a stale event snapshot', () => { + assert.throws( + () => resolveVerifiedBenchmarkBaseSha({ + override: '', + event: { + pull_request: { base: { sha: PR_BASE_SHA, ref: 'develop' } }, + }, + readLiveBaseSha: () => MOVED_BASE_SHA, + }), + /protected base moved/i, + ); +}); + +test('verified PR benchmark base refuses an override that differs from the live tip', () => { + assert.throws( + () => resolveVerifiedBenchmarkBaseSha({ + override: OVERRIDE_SHA, + event: { + pull_request: { base: { sha: PR_BASE_SHA, ref: 'develop' } }, + }, + readLiveBaseSha: () => PR_BASE_SHA, + }), + /override.*live protected base/i, + ); +}); + +test('verified benchmark base preserves push comparison semantics', () => { + assert.equal(resolveVerifiedBenchmarkBaseSha({ + override: '', + event: { before: PUSH_BEFORE_SHA, after: PUSH_AFTER_SHA }, + readLiveBaseSha: () => { + throw new Error('push runs must not compare event.before with the post-push live branch tip'); + }, + }), PUSH_BEFORE_SHA); +}); + +test('benchmark candidate prefers an explicit immutable override', () => { + assert.equal(resolveBenchmarkCandidateSha({ + override: HEAD_OVERRIDE_SHA, + event: { + pull_request: { head: { sha: PR_HEAD_SHA } }, + after: PUSH_AFTER_SHA, + }, + }), HEAD_OVERRIDE_SHA); +}); + +test('benchmark candidate uses the pull-request contributor head for pull_request runs', () => { + assert.equal(resolveBenchmarkCandidateSha({ + override: '', + event: { pull_request: { head: { sha: PR_HEAD_SHA } } }, + }), PR_HEAD_SHA); +}); + +test('benchmark candidate uses the pushed commit for push runs', () => { + assert.equal(resolveBenchmarkCandidateSha({ + override: '', + event: { after: PUSH_AFTER_SHA }, + }), PUSH_AFTER_SHA); +}); + +test('benchmark candidate fails closed instead of trusting the workflow worktree', () => { + assert.throws( + () => resolveBenchmarkCandidateSha({ override: '', event: {} }), + /benchmark candidate SHA is unavailable/i, + ); +}); + +test('benchmark candidate rejects malformed and all-zero revisions', () => { + for (const sha of ['not-a-sha', '0'.repeat(40)]) { + assert.throws( + () => resolveBenchmarkCandidateSha({ override: sha, event: {} }), + /benchmark candidate SHA is invalid/i, + ); + } +}); + +test('benchmark order measures each revision once in each execution position', () => { + assert.deepEqual(counterbalancedBenchmarkRounds(), [ + ['protected-base', 'candidate'], + ['candidate', 'protected-base'], + ]); +}); + +test('counterbalanced timing neutralizes a systematic second-run advantage', () => { + const measurements = [ + { label: 'protected-base', samples: Array(7).fill(10) }, + { label: 'candidate', samples: Array(7).fill(8) }, + { label: 'candidate', samples: Array(7).fill(10) }, + { label: 'protected-base', samples: Array(7).fill(8) }, + ]; + + const summary = summarizeCounterbalancedSamples(measurements); + assert.deepEqual(summary.baselineSamples, [ + ...Array(7).fill(10), + ...Array(7).fill(8), + ]); + assert.deepEqual(summary.candidateSamples, [ + ...Array(7).fill(8), + ...Array(7).fill(10), + ]); + assert.equal(summary.baselineMedianDurationMs, 9); + assert.equal(summary.candidateMedianDurationMs, 9); + assert.equal( + summary.improvementPercent, + 0, + 'execution-position speedup must not be misattributed to the candidate', + ); +});