From 213aa3628818bdb38babf7e2a1a8ff9f109a36c1 Mon Sep 17 00:00:00 2001 From: zhaowl94 <18801352315@163.com> Date: Fri, 31 Jul 2026 14:48:35 +0800 Subject: [PATCH] test: stabilize Lighthouse performance budgets --- package.json | 2 +- scripts/lighthouse-scores.mjs | 71 ++++++++++++++++++ scripts/run-lighthouse.mjs | 93 ++++++++++++++++++------ test/README.md | 7 +- test/unit/lighthouse-scores.test.mjs | 105 +++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 26 deletions(-) create mode 100644 scripts/lighthouse-scores.mjs create mode 100644 test/unit/lighthouse-scores.test.mjs diff --git a/package.json b/package.json index fde3e21..37369f5 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "check:static": "node test/scripts/check-static.mjs", "check:external-links": "node test/scripts/check-external-links.mjs", "check": "npm run format:check && npm run lint && npm run check:static", - "test:unit": "node --test --test-isolation=none test/unit/registration.test.mjs test/unit/static-checker.test.mjs", + "test:unit": "node --test --test-isolation=none test/unit/lighthouse-scores.test.mjs test/unit/registration.test.mjs test/unit/static-checker.test.mjs", "test:e2e": "playwright test --project=chromium --project=firefox --project=webkit --project=mobile-chromium --project=chrome --project=msedge", "test:e2e:linux": "playwright test --project=chromium --project=firefox --project=webkit --project=mobile-chromium --project=visual", "test:e2e:windows": "playwright test --project=chrome --project=msedge --project=firefox --project=mobile-chromium --workers=2", diff --git a/scripts/lighthouse-scores.mjs b/scripts/lighthouse-scores.mjs new file mode 100644 index 0000000..63f10de --- /dev/null +++ b/scripts/lighthouse-scores.mjs @@ -0,0 +1,71 @@ +export function median(values) { + if ( + !Array.isArray(values) || + values.length === 0 || + values.some((value) => !Number.isFinite(value)) + ) { + throw new TypeError("Median requires a non-empty array of finite numbers."); + } + + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 1) { + return sorted[midpoint]; + } + + return (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +export function medianScores(samples, categories) { + if (!Array.isArray(samples) || samples.length === 0) { + throw new TypeError("At least one Lighthouse score sample is required."); + } + + return Object.fromEntries( + categories.map((category) => [ + category, + median(samples.map((sample) => sample[category] ?? 0)), + ]), + ); +} + +export function scoresMeetThresholds(scores, thresholds) { + return Object.entries(thresholds).every( + ([category, minimum]) => + Number.isFinite(scores[category]) && scores[category] >= minimum, + ); +} + +export async function collectScoreSamples( + collectSample, + thresholds, + confirmationSampleCount = 3, +) { + if ( + typeof collectSample !== "function" || + !Number.isInteger(confirmationSampleCount) || + confirmationSampleCount < 1 + ) { + throw new TypeError( + "A collector and a positive sample count are required.", + ); + } + + const samples = [await collectSample(1)]; + + if (!scoresMeetThresholds(samples[0], thresholds)) { + for ( + let sampleNumber = 2; + sampleNumber <= confirmationSampleCount; + sampleNumber += 1 + ) { + samples.push(await collectSample(sampleNumber)); + } + } + + return { + samples, + scores: medianScores(samples, Object.keys(thresholds)), + }; +} diff --git a/scripts/run-lighthouse.mjs b/scripts/run-lighthouse.mjs index 7d14825..9841cd4 100644 --- a/scripts/run-lighthouse.mjs +++ b/scripts/run-lighthouse.mjs @@ -7,12 +7,14 @@ import { fileURLToPath } from "node:url"; import { chromium } from "@playwright/test"; import * as chromeLauncher from "chrome-launcher"; import lighthouse from "lighthouse"; +import { collectScoreSamples } from "./lighthouse-scores.mjs"; const projectRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), "..", ); -const reportDirectory = path.join(projectRoot, "test-results", "lighthouse"); +const testResultsDirectory = path.join(projectRoot, "test-results"); +const reportDirectory = path.join(testResultsDirectory, "lighthouse"); const serverScript = path.join( projectRoot, "node_modules", @@ -34,6 +36,8 @@ const thresholds = { performance: 0.95, seo: 0.95, }; +const auditTimeout = 90_000; +const confirmationSampleCount = 3; function startServer() { return spawn( @@ -86,12 +90,29 @@ function reportName(pagePath) { } async function runAudit(url, chromePort) { - const result = await lighthouse(url, { - logLevel: "error", - onlyCategories: Object.keys(thresholds), - output: ["html", "json"], - port: chromePort, + let timeoutHandle; + const timeout = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => { + reject( + new Error(`Lighthouse audit timed out after ${auditTimeout}ms: ${url}`), + ); + }, auditTimeout); }); + let result; + + try { + result = await Promise.race([ + lighthouse(url, { + logLevel: "error", + onlyCategories: Object.keys(thresholds), + output: ["html", "json"], + port: chromePort, + }), + timeout, + ]); + } finally { + clearTimeout(timeoutHandle); + } if (!result) { throw new Error(`Lighthouse did not return a report for ${url}.`); @@ -109,7 +130,35 @@ function scoresFor(lhr) { ); } +async function runAndSaveAudit(url, chromePort, name, sampleNumber) { + console.log(`${url}: starting Lighthouse sample ${sampleNumber}`); + const result = await runAudit(url, chromePort); + const [htmlReport, jsonReport] = result.report; + const suffix = sampleNumber === 1 ? "" : `.sample-${sampleNumber}`; + + await Promise.all([ + writeFile( + path.join(reportDirectory, `${name}${suffix}.report.html`), + htmlReport, + ), + writeFile( + path.join(reportDirectory, `${name}${suffix}.report.json`), + jsonReport, + ), + ]); + + return scoresFor(result.lhr); +} + async function main() { + if ( + path.dirname(reportDirectory) !== testResultsDirectory || + path.basename(reportDirectory) !== "lighthouse" + ) { + throw new Error(`Refusing to clean unexpected path: ${reportDirectory}`); + } + + await rm(reportDirectory, { force: true, recursive: true }); await mkdir(reportDirectory, { recursive: true }); const server = startServer(); @@ -136,23 +185,13 @@ async function main() { for (const pagePath of pagePaths) { const url = `${baseUrl}${pagePath}`; - const result = await runAudit(url, chrome.port); const name = reportName(pagePath); - const [htmlReport, jsonReport] = result.report; - - await Promise.all([ - writeFile( - path.join(reportDirectory, `${name}.report.html`), - htmlReport, - ), - writeFile( - path.join(reportDirectory, `${name}.report.json`), - jsonReport, - ), - ]); - - const scores = scoresFor(result.lhr); - summary[pagePath] = scores; + const { samples, scores } = await collectScoreSamples( + (sampleNumber) => runAndSaveAudit(url, chrome.port, name, sampleNumber), + thresholds, + confirmationSampleCount, + ); + summary[pagePath] = { samples, scores }; for (const [category, minimum] of Object.entries(thresholds)) { if (scores[category] < minimum) { @@ -168,11 +207,17 @@ async function main() { `${JSON.stringify(summary, null, 2)}\n`, ); - for (const [pagePath, scores] of Object.entries(summary)) { + for (const [pagePath, { samples, scores }] of Object.entries(summary)) { const formatted = Object.entries(scores) .map(([category, score]) => `${category}=${score.toFixed(2)}`) .join(", "); - console.log(`${pagePath}: ${formatted}`); + const sampleNote = + samples.length === 1 + ? "" + : `; median of ${samples.length} samples (${samples + .map((sample) => sample.performance.toFixed(2)) + .join(", ")} performance)`; + console.log(`${pagePath}: ${formatted}${sampleNote}`); } if (failures.length > 0) { diff --git a/test/README.md b/test/README.md index 86bbaf4..e5cc152 100644 --- a/test/README.md +++ b/test/README.md @@ -7,7 +7,7 @@ All repository verification lives under `test/`. Runtime reports are written to | Layer | Location | Purpose | | ----------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | | Static | `test/scripts/` | Compatibility paths, internal links, fragments, path case, CSP metadata, CSS parsing, local-only runtime assets, and payload budget | -| Unit | `test/unit/` | Registration helpers and static-checker utilities | +| Unit | `test/unit/` | Lighthouse scoring, registration helpers, and static-checker utilities | | Browser | `test/e2e/` | Page loading, navigation, forms, storage, resource origins, keyboard behavior, Axe, direct-file access, and screenshots | | Manual | `test/manual-accessibility.md` | Narrator, forced colors, keyboard, zoom, and Edge checks | | Performance | `scripts/run-lighthouse.mjs` | Performance, accessibility, best-practices, and SEO scores | @@ -34,6 +34,11 @@ npm.cmd run test:e2e:linux npm.cmd run test:lighthouse ``` +Every Lighthouse category keeps a `0.95` budget. A page is measured once when +its first sample passes. If any first score misses its budget, the runner saves +two confirmation samples and evaluates the three-run median. All raw reports +and scores remain in `test-results/lighthouse/`. + External link availability is intentionally non-blocking because remote sites and networks are unstable: ```powershell diff --git a/test/unit/lighthouse-scores.test.mjs b/test/unit/lighthouse-scores.test.mjs new file mode 100644 index 0000000..181a5fa --- /dev/null +++ b/test/unit/lighthouse-scores.test.mjs @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + collectScoreSamples, + median, + medianScores, + scoresMeetThresholds, +} from "../../scripts/lighthouse-scores.mjs"; + +test("median returns the middle score without mutating input", () => { + const values = [1, 0.94, 0.99]; + + assert.equal(median(values), 0.99); + assert.deepEqual(values, [1, 0.94, 0.99]); +}); + +test("median averages the two middle scores for an even sample", () => { + assert.equal(median([0.4, 0.8, 0.2, 0.6]), 0.5); +}); + +test("median rejects empty or non-finite samples", () => { + assert.throws(() => median([]), TypeError); + assert.throws(() => median([1, Number.NaN]), TypeError); +}); + +test("medianScores evaluates every requested Lighthouse category", () => { + const samples = [ + { accessibility: 1, performance: 0.94 }, + { accessibility: 0.98, performance: 1 }, + { accessibility: 0.99, performance: 0.97 }, + ]; + + assert.deepEqual(medianScores(samples, ["accessibility", "performance"]), { + accessibility: 0.99, + performance: 0.97, + }); +}); + +test("scoresMeetThresholds accepts equality and rejects low or missing scores", () => { + const thresholds = { accessibility: 0.95, performance: 0.95 }; + + assert.equal( + scoresMeetThresholds({ accessibility: 1, performance: 0.95 }, thresholds), + true, + ); + assert.equal( + scoresMeetThresholds({ accessibility: 1, performance: 0.94 }, thresholds), + false, + ); + assert.equal(scoresMeetThresholds({ accessibility: 1 }, thresholds), false); +}); + +test("collectScoreSamples stops after a passing first sample", async () => { + const sampleNumbers = []; + + const result = await collectScoreSamples( + async (sampleNumber) => { + sampleNumbers.push(sampleNumber); + return { performance: 1 }; + }, + { performance: 0.95 }, + ); + + assert.deepEqual(sampleNumbers, [1]); + assert.deepEqual(result, { + samples: [{ performance: 1 }], + scores: { performance: 1 }, + }); +}); + +test("collectScoreSamples confirms an initial miss with a three-run median", async () => { + const scoreSequence = [0.94, 1, 0.98]; + const sampleNumbers = []; + + const result = await collectScoreSamples( + async (sampleNumber) => { + sampleNumbers.push(sampleNumber); + return { performance: scoreSequence[sampleNumber - 1] }; + }, + { performance: 0.95 }, + ); + + assert.deepEqual(sampleNumbers, [1, 2, 3]); + assert.deepEqual(result, { + samples: [{ performance: 0.94 }, { performance: 1 }, { performance: 0.98 }], + scores: { performance: 0.98 }, + }); +}); + +test("collectScoreSamples keeps a persistent regression below the budget", async () => { + const scoreSequence = [0.94, 0.92, 0.96]; + + const result = await collectScoreSamples( + async (sampleNumber) => ({ + performance: scoreSequence[sampleNumber - 1], + }), + { performance: 0.95 }, + ); + + assert.equal(result.scores.performance, 0.94); + assert.equal( + scoresMeetThresholds(result.scores, { performance: 0.95 }), + false, + ); +});