Stabilize Lighthouse budgets with confirmation samples

package.json

@@ -19,7 +19,7 @@
     "check:static": "node test/scripts/check-static.mjs",
     "check:external-links": "node test/scripts/check-external-links.mjs",
     "check": "npm run format:check && npm run lint && npm run check:static",
-    "test:unit": "node --test --test-isolation=none test/unit/registration.test.mjs test/unit/static-checker.test.mjs",
+    "test:unit": "node --test --test-isolation=none test/unit/lighthouse-scores.test.mjs test/unit/registration.test.mjs test/unit/static-checker.test.mjs",
     "test:e2e": "playwright test --project=chromium --project=firefox --project=webkit --project=mobile-chromium --project=chrome --project=msedge",
     "test:e2e:linux": "playwright test --project=chromium --project=firefox --project=webkit --project=mobile-chromium --project=visual",
     "test:e2e:windows": "playwright test --project=chrome --project=msedge --project=firefox --project=mobile-chromium --workers=2",

scripts/lighthouse-scores.mjs

@@ -0,0 +1,71 @@
+export function median(values) {
+  if (
+    !Array.isArray(values) ||
+    values.length === 0 ||
+    values.some((value) => !Number.isFinite(value))
+  ) {
+    throw new TypeError("Median requires a non-empty array of finite numbers.");
+  }
+
+  const sorted = [...values].sort((left, right) => left - right);
+  const midpoint = Math.floor(sorted.length / 2);
+
+  if (sorted.length % 2 === 1) {
+    return sorted[midpoint];
+  }
+
+  return (sorted[midpoint - 1] + sorted[midpoint]) / 2;
+}
+
+export function medianScores(samples, categories) {
+  if (!Array.isArray(samples) || samples.length === 0) {
+    throw new TypeError("At least one Lighthouse score sample is required.");
+  }
+
+  return Object.fromEntries(
+    categories.map((category) => [
+      category,
+      median(samples.map((sample) => sample[category] ?? 0)),
+    ]),
+  );
+}
+
+export function scoresMeetThresholds(scores, thresholds) {
+  return Object.entries(thresholds).every(
+    ([category, minimum]) =>
+      Number.isFinite(scores[category]) && scores[category] >= minimum,
+  );
+}
+
+export async function collectScoreSamples(
+  collectSample,
+  thresholds,
+  confirmationSampleCount = 3,
+) {
+  if (
+    typeof collectSample !== "function" ||
+    !Number.isInteger(confirmationSampleCount) ||
+    confirmationSampleCount < 1
+  ) {
+    throw new TypeError(
+      "A collector and a positive sample count are required.",
+    );
+  }
+
+  const samples = [await collectSample(1)];
+
+  if (!scoresMeetThresholds(samples[0], thresholds)) {
+    for (
+      let sampleNumber = 2;
+      sampleNumber <= confirmationSampleCount;
+      sampleNumber += 1
+    ) {
+      samples.push(await collectSample(sampleNumber));
+    }
+  }
+
+  return {
+    samples,
+    scores: medianScores(samples, Object.keys(thresholds)),
+  };
+}

scripts/run-lighthouse.mjs

@@ -7,12 +7,14 @@ import { fileURLToPath } from "node:url";
 import { chromium } from "@playwright/test";
 import * as chromeLauncher from "chrome-launcher";
 import lighthouse from "lighthouse";
+import { collectScoreSamples } from "./lighthouse-scores.mjs";
 
 const projectRoot = path.resolve(
   path.dirname(fileURLToPath(import.meta.url)),
   "..",
 );
-const reportDirectory = path.join(projectRoot, "test-results", "lighthouse");
+const testResultsDirectory = path.join(projectRoot, "test-results");
+const reportDirectory = path.join(testResultsDirectory, "lighthouse");
 const serverScript = path.join(
   projectRoot,
   "node_modules",
@@ -34,6 +36,8 @@ const thresholds = {
   performance: 0.95,
   seo: 0.95,
 };
+const auditTimeout = 90_000;
+const confirmationSampleCount = 3;
 
 function startServer() {
   return spawn(
@@ -86,12 +90,29 @@ function reportName(pagePath) {
 }
 
 async function runAudit(url, chromePort) {
-  const result = await lighthouse(url, {
-    logLevel: "error",
-    onlyCategories: Object.keys(thresholds),
-    output: ["html", "json"],
-    port: chromePort,
+  let timeoutHandle;
+  const timeout = new Promise((_, reject) => {
+    timeoutHandle = setTimeout(() => {
+      reject(
+        new Error(`Lighthouse audit timed out after ${auditTimeout}ms: ${url}`),
+      );
+    }, auditTimeout);
   });
+  let result;
+
+  try {
+    result = await Promise.race([
+      lighthouse(url, {
+        logLevel: "error",
+        onlyCategories: Object.keys(thresholds),
+        output: ["html", "json"],
+        port: chromePort,
+      }),
+      timeout,
+    ]);
+  } finally {
+    clearTimeout(timeoutHandle);
+  }
 
   if (!result) {
     throw new Error(`Lighthouse did not return a report for ${url}.`);
@@ -109,7 +130,35 @@ function scoresFor(lhr) {
   );
 }
 
+async function runAndSaveAudit(url, chromePort, name, sampleNumber) {
+  console.log(`${url}: starting Lighthouse sample ${sampleNumber}`);
+  const result = await runAudit(url, chromePort);
+  const [htmlReport, jsonReport] = result.report;
+  const suffix = sampleNumber === 1 ? "" : `.sample-${sampleNumber}`;
+
+  await Promise.all([
+    writeFile(
+      path.join(reportDirectory, `${name}${suffix}.report.html`),
+      htmlReport,
+    ),
+    writeFile(
+      path.join(reportDirectory, `${name}${suffix}.report.json`),
+      jsonReport,
+    ),
+  ]);
+
+  return scoresFor(result.lhr);
+}
+
 async function main() {
+  if (
+    path.dirname(reportDirectory) !== testResultsDirectory ||
+    path.basename(reportDirectory) !== "lighthouse"
+  ) {
+    throw new Error(`Refusing to clean unexpected path: ${reportDirectory}`);
+  }
+
+  await rm(reportDirectory, { force: true, recursive: true });
   await mkdir(reportDirectory, { recursive: true });
 
   const server = startServer();
@@ -136,23 +185,13 @@ async function main() {
 
     for (const pagePath of pagePaths) {
       const url = `${baseUrl}${pagePath}`;
-      const result = await runAudit(url, chrome.port);
       const name = reportName(pagePath);
-      const [htmlReport, jsonReport] = result.report;
-
-      await Promise.all([
-        writeFile(
-          path.join(reportDirectory, `${name}.report.html`),
-          htmlReport,
-        ),
-        writeFile(
-          path.join(reportDirectory, `${name}.report.json`),
-          jsonReport,
-        ),
-      ]);
-
-      const scores = scoresFor(result.lhr);
-      summary[pagePath] = scores;
+      const { samples, scores } = await collectScoreSamples(
+        (sampleNumber) => runAndSaveAudit(url, chrome.port, name, sampleNumber),
+        thresholds,
+        confirmationSampleCount,
+      );
+      summary[pagePath] = { samples, scores };
 
       for (const [category, minimum] of Object.entries(thresholds)) {
         if (scores[category] < minimum) {
@@ -168,11 +207,17 @@ async function main() {
       `${JSON.stringify(summary, null, 2)}\n`,
     );
 
-    for (const [pagePath, scores] of Object.entries(summary)) {
+    for (const [pagePath, { samples, scores }] of Object.entries(summary)) {
       const formatted = Object.entries(scores)
         .map(([category, score]) => `${category}=${score.toFixed(2)}`)
         .join(", ");
-      console.log(`${pagePath}: ${formatted}`);
+      const sampleNote =
+        samples.length === 1
+          ? ""
+          : `; median of ${samples.length} samples (${samples
+              .map((sample) => sample.performance.toFixed(2))
+              .join(", ")} performance)`;
+      console.log(`${pagePath}: ${formatted}${sampleNote}`);
     }
 
     if (failures.length > 0) {

test/README.md

@@ -7,7 +7,7 @@ All repository verification lives under `test/`. Runtime reports are written to
 | Layer       | Location                       | Purpose                                                                                                                             |
 | ----------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- |
 | Static      | `test/scripts/`                | Compatibility paths, internal links, fragments, path case, CSP metadata, CSS parsing, local-only runtime assets, and payload budget |
-| Unit        | `test/unit/`                   | Registration helpers and static-checker utilities                                                                                   |
+| Unit        | `test/unit/`                   | Lighthouse scoring, registration helpers, and static-checker utilities                                                              |
 | Browser     | `test/e2e/`                    | Page loading, navigation, forms, storage, resource origins, keyboard behavior, Axe, direct-file access, and screenshots             |
 | Manual      | `test/manual-accessibility.md` | Narrator, forced colors, keyboard, zoom, and Edge checks                                                                            |
 | Performance | `scripts/run-lighthouse.mjs`   | Performance, accessibility, best-practices, and SEO scores                                                                          |
@@ -34,6 +34,11 @@ npm.cmd run test:e2e:linux
 npm.cmd run test:lighthouse
 ```
 
+Every Lighthouse category keeps a `0.95` budget. A page is measured once when
+its first sample passes. If any first score misses its budget, the runner saves
+two confirmation samples and evaluates the three-run median. All raw reports
+and scores remain in `test-results/lighthouse/`.
+
 External link availability is intentionally non-blocking because remote sites and networks are unstable:
 
 ```powershell

test/unit/lighthouse-scores.test.mjs

@@ -0,0 +1,105 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  collectScoreSamples,
+  median,
+  medianScores,
+  scoresMeetThresholds,
+} from "../../scripts/lighthouse-scores.mjs";
+
+test("median returns the middle score without mutating input", () => {
+  const values = [1, 0.94, 0.99];
+
+  assert.equal(median(values), 0.99);
+  assert.deepEqual(values, [1, 0.94, 0.99]);
+});
+
+test("median averages the two middle scores for an even sample", () => {
+  assert.equal(median([0.4, 0.8, 0.2, 0.6]), 0.5);
+});
+
+test("median rejects empty or non-finite samples", () => {
+  assert.throws(() => median([]), TypeError);
+  assert.throws(() => median([1, Number.NaN]), TypeError);
+});
+
+test("medianScores evaluates every requested Lighthouse category", () => {
+  const samples = [
+    { accessibility: 1, performance: 0.94 },
+    { accessibility: 0.98, performance: 1 },
+    { accessibility: 0.99, performance: 0.97 },
+  ];
+
+  assert.deepEqual(medianScores(samples, ["accessibility", "performance"]), {
+    accessibility: 0.99,
+    performance: 0.97,
+  });
+});
+
+test("scoresMeetThresholds accepts equality and rejects low or missing scores", () => {
+  const thresholds = { accessibility: 0.95, performance: 0.95 };
+
+  assert.equal(
+    scoresMeetThresholds({ accessibility: 1, performance: 0.95 }, thresholds),
+    true,
+  );
+  assert.equal(
+    scoresMeetThresholds({ accessibility: 1, performance: 0.94 }, thresholds),
+    false,
+  );
+  assert.equal(scoresMeetThresholds({ accessibility: 1 }, thresholds), false);
+});
+
+test("collectScoreSamples stops after a passing first sample", async () => {
+  const sampleNumbers = [];
+
+  const result = await collectScoreSamples(
+    async (sampleNumber) => {
+      sampleNumbers.push(sampleNumber);
+      return { performance: 1 };
+    },
+    { performance: 0.95 },
+  );
+
+  assert.deepEqual(sampleNumbers, [1]);
+  assert.deepEqual(result, {
+    samples: [{ performance: 1 }],
+    scores: { performance: 1 },
+  });
+});
+
+test("collectScoreSamples confirms an initial miss with a three-run median", async () => {
+  const scoreSequence = [0.94, 1, 0.98];
+  const sampleNumbers = [];
+
+  const result = await collectScoreSamples(
+    async (sampleNumber) => {
+      sampleNumbers.push(sampleNumber);
+      return { performance: scoreSequence[sampleNumber - 1] };
+    },
+    { performance: 0.95 },
+  );
+
+  assert.deepEqual(sampleNumbers, [1, 2, 3]);
+  assert.deepEqual(result, {
+    samples: [{ performance: 0.94 }, { performance: 1 }, { performance: 0.98 }],
+    scores: { performance: 0.98 },
+  });
+});
+
+test("collectScoreSamples keeps a persistent regression below the budget", async () => {
+  const scoreSequence = [0.94, 0.92, 0.96];
+
+  const result = await collectScoreSamples(
+    async (sampleNumber) => ({
+      performance: scoreSequence[sampleNumber - 1],
+    }),
+    { performance: 0.95 },
+  );
+
+  assert.equal(result.scores.performance, 0.94);
+  assert.equal(
+    scoresMeetThresholds(result.scores, { performance: 0.95 }),
+    false,
+  );
+});