diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 3f518164..0b378e65 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -610,6 +610,71 @@ jobs: path: .ci-artifacts/frontend-next-unit-junit.xml if-no-files-found: warn retention-days: 7 + # #3318: this tier collected no coverage at all, so there was no + # baseline and nothing here could catch a change that deleted tested + # behaviour. The tests that remained would keep passing over code that + # had stopped being tested, and the job would stay green -- the tests + # that assert the deleted behaviour would be the thing a reviewer has + # to notice is missing. + # + # Two gates, both compared against the committed + # arcane/home/honeypot-dashboard/frontend-next/coverage-baseline.json + # and both computed from this run's own measurement. Neither is a + # threshold somebody chose: + # + # coverage:ratchet the non-regression floor. The baseline is what the + # suite measures today -- 806/7359 lines (10.95%) + # and 346/7250 branches (4.77%) across 127 files of + # src/ -- measured on this job's own #3331 node:22 + # image. It is a floor, not a target, and the number + # in the file is a measurement rather than an + # aspiration on purpose: an importable 60% would + # have made this gate red on the day it landed and + # bought nothing. Raising it is a separate, + # deliberate commit. + # test:discovery a test-shaped file that no configured runner + # collects. Its own step so a coverage failure + # cannot hide a discovery failure. + # + # `npm test` above is untouched and stays uninstrumented: a second full + # run of the suite is the honest price of keeping the command deploy.yml, + # the README and a developer's own loop free of coverage, and it is + # cheap here -- 6.9s uninstrumented against 6.9s with the v8 provider on + # the image this job runs (the cost is transform/import, not + # instrumentation), against a 45-minute budget. + # + # `set -euo pipefail` so a failing coverage run cannot be followed by a + # ratchet that then reports on a missing report and takes the blame for + # it. + - name: "Coverage report and ratchet (#3318)" + working-directory: arcane/home/honeypot-dashboard/frontend-next + run: | + set -euo pipefail + npm run test:coverage + npm run coverage:ratchet + - name: "Test discovery guard (#3318)" + working-directory: arcane/home/honeypot-dashboard/frontend-next + run: npm run test:discovery + # Same per-(run, attempt) name as the JUnit upload above, for the same + # reason: run_id alone still collides on a re-run of the same run, and a + # name two uploads share is not "newest wins" under v4 -- it is the + # second uploader deleting the first one's evidence. No overwrite here + # either, and the retention matches. + # + # `coverage/` is named as a directory rather than file-by-file because + # it is not hidden -- the .ci-artifacts caveat that forces the JUnit + # upload to name its file does not apply, and a directory keeps the + # report extensible (adding the lcov html view later is a config change + # here, not a workflow change). The contents stay a closed set: only the + # files the coverage reporter wrote, from the run above, can land here. + - name: Upload unit test coverage + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: frontend-next-coverage-${{ github.run_id }}-${{ github.run_attempt }} + path: arcane/home/honeypot-dashboard/frontend-next/coverage/ + if-no-files-found: warn + retention-days: 7 # No live-ES smoke suite here on purpose: port-tests/ needs the real # cluster over an SSH tunnel to the homeserver (see its README) -- # not reachable from a GitHub-hosted runner, and not appropriate to @@ -699,6 +764,30 @@ jobs: path: .ci-artifacts/frontend-next-unit-junit.xml if-no-files-found: warn retention-days: 7 + # #3318: twin of the homeserver copy's coverage + ratchet + discovery + # steps. Present here for the reason the header comment gives: steps stay + # byte-identical across the twins so a red result means the same thing + # wherever it ran. A ratchet that only ran on the self-hosted executor + # would let a regression through on every degraded day, which is exactly + # the day the fallback twin exists for. (Full rationale, and the measured + # baseline, live on the homeserver copy above.) + - name: "Coverage report and ratchet (#3318)" + working-directory: arcane/home/honeypot-dashboard/frontend-next + run: | + set -euo pipefail + npm run test:coverage + npm run coverage:ratchet + - name: "Test discovery guard (#3318)" + working-directory: arcane/home/honeypot-dashboard/frontend-next + run: npm run test:discovery + - name: Upload unit test coverage + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: frontend-next-coverage-${{ github.run_id }}-${{ github.run_attempt }} + path: arcane/home/honeypot-dashboard/frontend-next/coverage/ + if-no-files-found: warn + retention-days: 7 # No live-ES smoke suite here on purpose -- see the homeserver twin. - run: npm run build working-directory: arcane/home/honeypot-dashboard/frontend-next diff --git a/arcane/home/honeypot-dashboard/frontend-next/.gitignore b/arcane/home/honeypot-dashboard/frontend-next/.gitignore index c1ce433d..0ef93362 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/.gitignore +++ b/arcane/home/honeypot-dashboard/frontend-next/.gitignore @@ -25,3 +25,10 @@ playwright-report/ # there, never committed -- a committed mutation report is a stale report. reports/mutation/ .stryker-tmp/ + +# #3318: the v8 coverage report (coverage-summary.json, lcov.info and the html +# report). Same reasoning as the two blocks above: regenerated on every +# `npm run test:coverage`, uploaded to CI from there, and never committed -- +# coverage-baseline.json at the package root is the committed record, and a +# committed report would be a second, stale one. +coverage/ diff --git a/arcane/home/honeypot-dashboard/frontend-next/README.md b/arcane/home/honeypot-dashboard/frontend-next/README.md index db95e387..343a6caf 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/README.md +++ b/arcane/home/honeypot-dashboard/frontend-next/README.md @@ -75,6 +75,46 @@ for how a commit actually reaches the live host. `backend-api.sh`, `bff-load.sh`) — see `../port-tests/README.md`. Run against a real build, not `npm run dev`'s HMR server. +### Unit tests, coverage and the ratchet (#3318) + +`npm test` is the unit suite and stays uninstrumented — no coverage, no +report tree — so it stays the cheap command that `deploy.yml`, this README +and your own loop all reach for. The coverage number lives in its own +command: + +```bash +npm run test:coverage # vitest + v8 coverage for src/, into coverage/ (gitignored) +npm run coverage:ratchet # compare that measurement to the committed baseline; non-zero on a drop +npm run test:discovery # fail if a *.test.ts / *.spec.ts exists that no runner collects +``` + +`coverage-baseline.json` at this directory's root is the committed +non-regression floor: **806/7359 lines (10.95%) and 346/7250 branches +(4.77%)** across 127 files of `src/`, measured on the `node:22` image the +tier ships. It is a measurement, not a target, and the low number is the +real one — the tier's tests concentrate on `src/lib` server logic, and most +of `src/routes` and `src/components` is covered by the Playwright matrix +instead, which the v8 provider does not see. Do not hand-edit it, and do +not regenerate it to turn a red run green. + +To move it on purpose, having first established the drop is intended +rather than a lost test: + +```bash +npm run test:coverage && npm run coverage:baseline # commit both, together +``` + +The ratchet has two independent gates. `tolerance.coveredCount` is 0: no +covered line or branch may stop being covered, which is what catches a +change that deletes tested behaviour, and which no shrinking-denominator +trick can satisfy. `tolerance.pctPoints` is 1.0: the overall percentage may +not fall more than that, which catches a change that adds a lot of untested +source. At a ~11% baseline the second is the coarse one — it needs several +hundred new uncovered lines to move — which is the honest shape of a +low-coverage ratchet and the reason the first one carries the weight. +`scripts/tests/test_3318_coverage_ratchet.py` in the repository root pins +both, so neither can be loosened without that test going red. + ## New-page review checklist (capped-truth discipline, #2179) When authoring or reviewing a page, check the ways a number can quietly lie. diff --git a/arcane/home/honeypot-dashboard/frontend-next/coverage-baseline.json b/arcane/home/honeypot-dashboard/frontend-next/coverage-baseline.json new file mode 100644 index 00000000..426760da --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/coverage-baseline.json @@ -0,0 +1,42 @@ +{ + "about": "Generated measurement, not a target. Do not hand-edit these numbers and do not regenerate them to make a red run green -- coverage-baseline.json exists so that coverage cannot fall without a commit that says it did. Update only via `npm run coverage:baseline`, in a commit whose diff shows the change.", + "recordedAt": "2026-09-27T16:54:00.850Z", + "measuredWith": { + "node": "v22.23.2", + "vitest": "4.1.11", + "coverageProvider": "@vitest/coverage-v8 4.1.11", + "note": "measured on the runtime the image ships (node:22-alpine), which is the runtime CI runs" + }, + "scope": "src/**/*.{ts,tsx} minus *.test.*, *.spec.*, *.d.ts and src/routeTree.gen.ts (vitest.config.ts coverage.include/exclude)", + "tolerance": { + "pctPoints": 1, + "coveredCount": 0 + }, + "toleranceWhy": { + "pctPoints": "percentage points of lines/branches. This is the half that fires when a change ADDS untested source. 1.0pp of a 7359-line denominator is ~74 lines, about 1.3 of src/'s average 58-line file.", + "coveredCount": "raw covered lines/branches allowed to disappear; 0 is deliberate. This is the half that fires when a change REMOVES tested behaviour, and comparing raw counts cannot be satisfied by deleting an untested file to shrink the denominator." + }, + "files": 127, + "totals": { + "lines": { + "covered": 806, + "total": 7359, + "pct": 10.9526 + }, + "branches": { + "covered": 346, + "total": 7250, + "pct": 4.7724 + }, + "statements": { + "covered": 859, + "total": 8452, + "pct": 10.1633 + }, + "functions": { + "covered": 107, + "total": 2666, + "pct": 4.0135 + } + } +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/package-lock.json b/arcane/home/honeypot-dashboard/frontend-next/package-lock.json index 3ee1a4a4..f5fdd041 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/package-lock.json +++ b/arcane/home/honeypot-dashboard/frontend-next/package-lock.json @@ -31,6 +31,7 @@ "@types/react": "^19.2.0", "@types/react-dom": "^19.2.5", "@vitejs/plugin-react": "^6.1.1", + "@vitest/coverage-v8": "^5.0.1", "fast-check": "^4.10.2", "jsdom": "^30.0.1", "lru-cache": "^11.2.6", @@ -760,6 +761,16 @@ "node": ">=6.9.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@bramus/specificity": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", @@ -3307,15 +3318,67 @@ } } }, + "node_modules/@vitest/coverage-v8": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-5.0.2.tgz", + "integrity": "sha512-3ffHBEi8DOOBLwIGBhOBZbRfYFYWjMUuxZicONdhuFEFEG5AVsMOSrVeuROskqnqpbOmEJwxM2eTVZ9N3a1t+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/istanbul-lib-coverage": "^1.0.0", + "@vitest/istanbul-lib-report": "^1.0.0", + "ast-v8-to-istanbul": "^1.0.5", + "magicast": "^0.5.4", + "obug": "^2.1.4", + "std-env": "^4.2.0", + "tinyrainbow": "^3.1.1" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "5.0.2", + "vitest": "5.0.2" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, + "node_modules/@vitest/istanbul-lib-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@vitest/istanbul-lib-coverage/-/istanbul-lib-coverage-1.0.2.tgz", + "integrity": "sha512-9J/JMwOf9AoJhAywhrn7ScKTL38hsWQP/qPG60OtaAFcQ5OXPwKsxZFlbnuCKmZ61m8/lGgHYnFpdyQZUvG/iA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22" + } + }, + "node_modules/@vitest/istanbul-lib-report": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@vitest/istanbul-lib-report/-/istanbul-lib-report-1.0.2.tgz", + "integrity": "sha512-gUsfXZJbzPamoIY5TvHFiMMoXESBrUMo+xqaj+rYrWI69+EnvRlYBlP96ZnHPY4vX8kyUpgAnFUCg5wZG/HkDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/istanbul-lib-coverage": "1.0.2" + }, + "engines": { + "node": ">=22" + } + }, "node_modules/@vitest/mocker": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.1.tgz", - "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.2.tgz", + "integrity": "sha512-Z5FS00Q1SJHkB35xATsmWGdQ5WA1/0MV3CDjqyv7GavHv1OfOj145MNfHOlHk7QLes21dKFDHr8EO2zvL+9WGA==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.31", - "@vitest/spy": "5.0.1", + "@vitest/spy": "5.0.2", "estree-walker": "^3.0.3", "magic-string": "^1.2.3" }, @@ -3346,9 +3409,9 @@ } }, "node_modules/@vitest/spy": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.1.tgz", - "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.2.tgz", + "integrity": "sha512-Ijc7T1nT9efNb5LxvjaBrEqw3f/QwUv5EE0nKqZxgqsaV/FxAAZ8baGylA8X/Z2oS4Lp+K74Jr6dTJsDKxJDeg==", "dev": true, "license": "MIT", "funding": { @@ -3448,6 +3511,25 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.7.tgz", + "integrity": "sha512-kFL68AG6ajd8fg248zwM9GQrUWEp79gsmjum34OEXjs4yHuUMZfYKwOLW9GMmB4oNvVrj+EAGxsP7ye2UR9UlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, + "node_modules/ast-v8-to-istanbul/node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/babel-dead-code-elimination": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/babel-dead-code-elimination/-/babel-dead-code-elimination-1.0.12.tgz", @@ -4995,6 +5077,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.5.tgz", + "integrity": "sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -5920,13 +6014,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true, - "license": "ISC" - }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -5971,13 +6058,6 @@ "node": ">=20.16.0" } }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", - "dev": true, - "license": "MIT" - }, "node_modules/standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", @@ -6040,9 +6120,9 @@ "license": "MIT" }, "node_modules/tinybench": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.1.4.tgz", - "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.0.tgz", + "integrity": "sha512-78U2TlB2CnVenajOFzf3BKSm0J6oz5L0NV7g32LCPccvYc0lbWvys4d3uUUCS2B1N8PAf2+aekR8i1KbC3HO7Q==", "dev": true, "license": "MIT", "engines": { @@ -6075,6 +6155,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tldts": { "version": "7.4.11", "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.11.tgz", @@ -6464,14 +6554,14 @@ } }, "node_modules/vitest": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.1.tgz", - "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.2.tgz", + "integrity": "sha512-7MQrx9pDv5aHiUcovIb/70Ys3tgtkUVgCtledvKdCmEO+/1Dicq5ZqoSxOW034m03oqC+oHOKui2dM6qtMLoJg==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/mocker": "5.0.1", + "@vitest/mocker": "5.0.2", "chai": "^6.2.2", "es-module-lexer": "^2.3.2", "expect-type": "^1.4.0", @@ -6479,10 +6569,10 @@ "obug": "^2.1.4", "picomatch": "^4.0.7", "std-env": "^4.2.0", - "tinybench": "6.1.4", - "tinyexec": "1.3.0", + "tinybench": "^6.1.4", + "tinyexec": "^1.3.0", "tinyglobby": "^0.2.17", - "why-is-node-running": "^2.3.0" + "why-is-node-running": "^3.2.1" }, "bin": { "vitest": "vitest.mjs" @@ -6497,12 +6587,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "5.0.1", - "@vitest/browser-preview": "5.0.1", + "@vitest/browser-playwright": "5.0.2", + "@vitest/browser-preview": "5.0.2", "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", - "@vitest/coverage-istanbul": "5.0.1", - "@vitest/coverage-v8": "5.0.1", - "@vitest/ui": "5.0.1", + "@vitest/coverage-istanbul": "5.0.2", + "@vitest/coverage-v8": "5.0.2", + "@vitest/ui": "5.0.2", "happy-dom": "*", "jsdom": "*", "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" @@ -6634,20 +6724,16 @@ } }, "node_modules/why-is-node-running": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", - "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.2.tgz", + "integrity": "sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A==", "dev": true, "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, "bin": { "why-is-node-running": "cli.js" }, "engines": { - "node": ">=8" + "node": ">=20.11" } }, "node_modules/wrap-ansi": { diff --git a/arcane/home/honeypot-dashboard/frontend-next/package.json b/arcane/home/honeypot-dashboard/frontend-next/package.json index ce1b4b2b..bc9d1d4d 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/package.json +++ b/arcane/home/honeypot-dashboard/frontend-next/package.json @@ -16,6 +16,10 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", + "test:coverage": "vitest run --coverage", + "test:discovery": "node scripts/test-discovery-guard.mjs", + "coverage:ratchet": "node scripts/coverage-ratchet.mjs", + "coverage:baseline": "node scripts/coverage-ratchet.mjs --update", "test:browser": "playwright test", "test:property": "vitest run src/lib/appearanceCookie.property.test.ts", "test:mutation": "stryker run" @@ -46,6 +50,7 @@ "@types/react": "^19.2.0", "@types/react-dom": "^19.2.5", "@vitejs/plugin-react": "^6.1.1", + "@vitest/coverage-v8": "^5.0.1", "fast-check": "^4.10.2", "jsdom": "^30.0.1", "lru-cache": "^11.2.6", diff --git a/arcane/home/honeypot-dashboard/frontend-next/scripts/coverage-ratchet.mjs b/arcane/home/honeypot-dashboard/frontend-next/scripts/coverage-ratchet.mjs new file mode 100644 index 00000000..16fa0dcb --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/scripts/coverage-ratchet.mjs @@ -0,0 +1,315 @@ +#!/usr/bin/env node +// #3318: the non-regression ratchet for this tier's line and branch coverage. +// +// Why this is a separate script and not vitest's own `coverage.thresholds`: +// a threshold in the config is a number somebody typed, and typing it is how a +// gate gets set to whatever would make it green. The number this file checks +// against lives in coverage-baseline.json, which is committed, which is a +// generated measurement rather than a claim, and whose only way to change is +// `npm run coverage:baseline` -- a diff a reviewer can see. There is no +// autoUpdate path here on purpose: vitest's `thresholds.autoUpdate` would +// rewrite the very file the check reads, which is a check that can rewrite +// itself, which is not a check. +// +// What it asserts, and why each half exists: +// +// 1. lines.pct and branches.pct must not drop more than +// tolerance.pctPoints below the baseline. This is the issue's actual +// contract -- "fails when coverage drops beyond a small tolerance" -- and +// it is the half that fires when a change ADDS untested source: this +// package's src/ files average 58 lines, so one percentage point of a +// 7359-line denominator is ~74 lines, about 1.3 average files. A PR +// landing a new route does not fail on this; a PR landing several does, +// which is worth a conversation rather than silence. +// +// 2. covered lines and covered branches must not drop by ANY amount +// (tolerance.coveredCount is 0). This is the half that fires when a +// change REMOVES tested behaviour, and it is why the ratchet cannot be +// gamed by the obvious move: deleting an untested source file shrinks the +// denominator and makes pct go UP, which half 1 alone would wave through. +// Comparing raw covered counts cannot be gamed that way. +// +// Both are computed from covered/total in coverage-summary.json rather than +// from its `pct` field, which istanbul rounds to 2 decimals -- a rounding +// artefact of 0.01pp is noise next to a 1.0pp tolerance, but the raw counts +// are the measurement and the percentage is derived from it, so that is the +// order the arithmetic goes in. +// +// Usage: +// node scripts/coverage-ratchet.mjs check (default). Exits 1 on a +// regression, 0 otherwise. +// node scripts/coverage-ratchet.mjs --update rewrite coverage-baseline.json +// from the current measurement. +// Deliberate, and a visible diff. +// +// Reads the last `npm run test:coverage` report by default; --summary +// points it elsewhere, which is what makes the failure paths testable without +// a coverage run. +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' + +const PACKAGE_ROOT = resolve(import.meta.dirname, '..') +const SUMMARY_PATH = 'coverage/coverage-summary.json' +const BASELINE_PATH = 'coverage-baseline.json' + +// The two metrics the issue names. statements and functions are recorded in +// the baseline as context and are NOT gated: the contract is line and branch +// coverage, and adding a gate nobody agreed to is how a ratchet stops being +// the thing that was asked for. +const GATED = ['lines', 'branches'] +// For the file-path note only -- the summary's keys are relative to whichever +// directory the report was written from, which differs between a local run and +// a container mount, so nothing downstream may key off them. +const RECORDED = [...GATED, 'statements', 'functions'] + +function parseArgs(argv) { + const opts = { update: false, summary: SUMMARY_PATH } + for (let i = 0; i < argv.length; i += 1) { + if (argv[i] === '--update') opts.update = true + else if (argv[i] === '--summary') { + const value = argv[i + 1] + if (!value) { + console.error('coverage-ratchet: --summary needs a file path') + process.exit(2) + } + opts.summary = value + i += 1 + } else { + console.error(`coverage-ratchet: unknown argument: ${argv[i]}`) + process.exit(2) + } + } + return opts +} + +function readJson(path, what) { + const full = resolve(PACKAGE_ROOT, path) + let raw + try { + raw = readFileSync(full, 'utf8') + } catch { + // A missing measurement is a failure, not a pass. Silently passing here + // would make the whole gate conditional on a report that may not exist. + console.error(`coverage-ratchet: cannot read the ${what} at ${full}`) + console.error(` run \`npm run test:coverage\` first (it writes ${SUMMARY_PATH})`) + process.exit(2) + } + try { + return JSON.parse(raw) + } catch (err) { + console.error(`coverage-ratchet: ${full} is not valid JSON: ${err.message}`) + process.exit(2) + } +} + +// The one case where "absent" is not an error: bootstrapping the very first +// baseline. Everything else treats an absent baseline as a failure, because a +// check with no baseline to compare against is a check that never fires. +function readBaselineIfPresent() { + if (!existsSync(resolve(PACKAGE_ROOT, BASELINE_PATH))) return null + return readJson(BASELINE_PATH, 'baseline') +} + +function fileCount(summary) { + return Object.keys(summary).filter((k) => k !== 'total').length +} + +function measure(summary) { + const totals = {} + for (const metric of RECORDED) { + const t = summary.total?.[metric] + if (!t || !Number.isFinite(t.total) || !Number.isFinite(t.covered)) { + console.error(`coverage-ratchet: coverage-summary.json has no usable total.${metric}`) + console.error(' if the file was hand-edited, regenerate it: npm run test:coverage') + process.exit(2) + } + totals[metric] = { covered: t.covered, total: t.total, pct: round(100 * t.covered / t.total) } + } + return totals +} + +function round(pct) { + return Math.round(pct * 1e4) / 1e4 +} + +function describeRuntime() { + let vitest = 'unknown' + try { + vitest = JSON.parse(readFileSync(resolve(PACKAGE_ROOT, 'node_modules/vitest/package.json'), 'utf8')).version + } catch { + // Not installed (someone ran this outside an install). Not fatal on its + // own -- the node line below is the one that changes the numbers. + } + let coverage = 'unknown' + try { + coverage = JSON.parse(readFileSync(resolve(PACKAGE_ROOT, 'node_modules/@vitest/coverage-v8/package.json'), 'utf8')).version + } catch { + // as above + } + return { node: process.version, vitest, coverageProvider: `@vitest/coverage-v8 ${coverage}` } +} + +function validateBaseline(baseline) { + const problems = [] + for (const key of ['recordedAt', 'measuredWith', 'tolerance', 'totals', 'files']) { + if (baseline?.[key] === undefined) problems.push(`missing "${key}"`) + } + for (const metric of GATED) { + if (!baseline?.totals?.[metric]) { + problems.push(`missing "totals.${metric}"`) + continue + } + for (const field of ['covered', 'total', 'pct']) { + if (!Number.isFinite(baseline.totals[metric][field])) problems.push(`"totals.${metric}.${field}" is not a number`) + } + } + if (!Number.isFinite(baseline?.tolerance?.pctPoints)) problems.push('missing "tolerance.pctPoints"') + if (!Number.isFinite(baseline?.tolerance?.coveredCount)) problems.push('missing "tolerance.coveredCount"') + if (problems.length) { + console.error('coverage-ratchet: coverage-baseline.json is not a usable baseline:') + for (const p of problems) console.error(` - ${p}`) + console.error(' regenerate it deliberately and commit the result:') + console.error(' npm run test:coverage && npm run coverage:baseline') + process.exit(2) + } +} + +function writeBaseline(summary) { + const totals = measure(summary) + const doc = { + // JSON has no comments, so the contract lives in the one key every JSON + // reader tolerates. It is a string because a value nobody reads is a + // value nobody maintains. + about: + 'Generated measurement, not a target. Do not hand-edit these numbers and do not ' + + 'regenerate them to make a red run green -- coverage-baseline.json exists so that ' + + 'coverage cannot fall without a commit that says it did. Update only via ' + + '`npm run coverage:baseline`, in a commit whose diff shows the change.', + recordedAt: new Date().toISOString(), + measuredWith: { + ...describeRuntime(), + // Stated because it decides the denominator: v8 coverage counts are a + // property of the Node that ran the tests, so a Node major bump can move + // these numbers with no source change at all. + note: 'measured on the runtime the image ships (node:22-alpine), which is the runtime CI runs', + }, + scope: 'src/**/*.{ts,tsx} minus *.test.*, *.spec.*, *.d.ts and src/routeTree.gen.ts (vitest.config.ts coverage.include/exclude)', + tolerance: { + // Percentage points, lines and branches. 1.0pp of this denominator is + // ~74 lines, about 1.3 of src/'s average 58-line file: one new route does + // not trip it, a batch of them does. + pctPoints: 1.0, + // Raw covered lines/branches allowed to disappear. Zero on purpose -- + // this is the half that catches deleted tested behaviour, and it is + // immune to the "delete an untested file to raise the percentage" move. + // A PR that legitimately removes covered code updates this file in the + // same commit; that is the intended cost. + coveredCount: 0, + }, + // Repeated here, on the artifact a reviewer actually reads in the diff. + // The same reasoning is in scripts/coverage-ratchet.mjs next to the code + // that enforces it; a baseline nobody can interpret is a baseline nobody + // will notice being edited by hand. + toleranceWhy: { + pctPoints: + 'percentage points of lines/branches. This is the half that fires when a change ADDS untested ' + + 'source. 1.0pp of a 7359-line denominator is ~74 lines, about 1.3 of src/\'s average 58-line file.', + coveredCount: + 'raw covered lines/branches allowed to disappear; 0 is deliberate. This is the half that fires ' + + 'when a change REMOVES tested behaviour, and comparing raw counts cannot be satisfied by deleting ' + + 'an untested file to shrink the denominator.', + }, + files: fileCount(summary), + totals, + } + const target = resolve(PACKAGE_ROOT, BASELINE_PATH) + writeFileSync(target, `${JSON.stringify(doc, null, 2)}\n`, 'utf8') + console.log(`coverage-ratchet: wrote ${target}`) + for (const metric of GATED) { + const t = totals[metric] + console.log(` ${metric.padEnd(9)} ${t.covered}/${t.total} = ${t.pct}%`) + } + console.log(' this is now the floor. Commit it deliberately -- it is the record of what the') + console.log(' suite covers today, not a claim about what it ought to cover.') +} + +function check(opts) { + const summary = readJson(opts.summary, 'coverage summary') + const baseline = readJson(BASELINE_PATH, 'baseline') + validateBaseline(baseline) + const totals = measure(summary) + const files = fileCount(summary) + const now = describeRuntime() + const then = baseline.measuredWith ?? {} + + // A note, not a failure. A Node major bump changes v8's counters, and the + // first thing a reader needs is that fact rather than a percentage that + // moved for a reason that has nothing to do with their diff. + if (then.node && now.node !== then.node) { + console.log(`coverage-ratchet: NOTE runtime differs from the baseline -- baseline ${then.node}, now ${now.node}.`) + console.log(' A Node major can change v8 coverage counts with no source change; if this fails,') + console.log(' check the runtime before assuming the diff caused it.') + } + if (then.vitest && now.vitest !== then.vitest) { + console.log(`coverage-ratchet: NOTE vitest differs from the baseline -- baseline ${then.vitest}, now ${now.vitest}.`) + } + if (typeof baseline.files === 'number' && baseline.files !== files) { + console.log(`coverage-ratchet: NOTE scope changed -- baseline covers ${baseline.files} files, this run ${files}.`) + console.log(' A file entering or leaving src/ moves the denominator on its own; judge that diff, not this number.') + } + + const failures = [] + console.log(`coverage-ratchet: ${files} files under src/ (baseline ${baseline.files})`) + for (const metric of GATED) { + const got = totals[metric] + const want = baseline.totals[metric] + const dropPct = want.pct - got.pct + const dropCovered = want.covered - got.covered + const pctOk = dropPct <= baseline.tolerance.pctPoints + 1e-9 + const coveredOk = dropCovered <= baseline.tolerance.coveredCount + const mark = pctOk && coveredOk ? 'ok ' : 'FAIL' + console.log( + ` ${mark} ${metric.padEnd(9)} ${got.pct}% (${got.covered}/${got.total})` + + ` baseline ${want.pct}% (${want.covered}/${want.total})` + + ` drop ${round(dropPct)}pp / ${dropCovered} covered`, + ) + if (!pctOk) { + failures.push( + `${metric} coverage fell ${round(dropPct)}pp, past the ${baseline.tolerance.pctPoints}pp tolerance ` + + `(${got.pct}% now, ${want.pct}% in coverage-baseline.json)`, + ) + } + if (!coveredOk) { + failures.push( + `${metric} lost ${dropCovered} covered ${metric === 'lines' ? 'line' : 'branch'}(es) with no tolerance allowed ` + + `(${got.covered} now, ${want.covered} in coverage-baseline.json)`, + ) + } + } + + if (failures.length) { + console.error('') + console.error('::error::coverage regression (#3318) -- this tier collects no other evidence that tested behaviour still runs:') + for (const f of failures) console.error(` - ${f}`) + console.error('') + console.error('This is a non-regression floor, not a target. To move it on purpose -- having first') + console.error('established that the drop is intended rather than a lost test -- run:') + console.error(' npm run test:coverage && npm run coverage:baseline') + console.error('and commit coverage-baseline.json in the same commit as the change that caused the drop.') + process.exit(1) + } + console.log('coverage-ratchet: no regression. The baseline is a floor, not a target -- raising it is a separate, deliberate commit.') +} + +const opts = parseArgs(process.argv.slice(2)) +if (opts.update) { + // --update still reads the baseline first, so an update cannot silently + // replace something that was not a usable baseline (a hand-written one, a + // truncated one, a half-merged one) without the shape check noticing. + const existing = readBaselineIfPresent() + if (existing) validateBaseline(existing) + else console.log('coverage-ratchet: no coverage-baseline.json yet -- bootstrapping the first one.') + writeBaseline(readJson(opts.summary, 'coverage summary')) +} else { + check(opts) +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/scripts/test-discovery-guard.mjs b/arcane/home/honeypot-dashboard/frontend-next/scripts/test-discovery-guard.mjs new file mode 100644 index 00000000..15749b4a --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/scripts/test-discovery-guard.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +// #3318: fail when a test-shaped file exists that no configured runner collects. +// +// The failure this exists to catch is silent. A test file is named like a test +// file, sits next to the code it covers, and asserts real behaviour -- and +// nothing runs it. It is worse than no test at all, because the file is the +// evidence: a reader, and every future coverage number, counts it as a test +// that guards the module. The usual way in is a move, not a mistake. Somebody +// puts a spec next to a route, or a test in a directory the include globs do +// not reach, and the suite stays green because the file was never collected. +// +// So the check is the other direction from "is the suite passing": it asks what +// the runners would actually collect, and compares that against what is on +// disk. Both halves come from the runners themselves -- `vitest list +// --filesOnly` and `playwright test --list` -- rather than from a second +// implementation of their glob semantics here. A hand-rolled matcher against +// vitest.config.ts's include globs would agree with vitest until the day it +// did not, and the day it did not would be the day it reported a healthy tree +// over a test that had stopped running. +// +// Two runners, because two runners own .ts/.spec.ts in this package: +// vitest's include globs (src/**) and Playwright's testDir (e2e/). Asking only +// vitest would flag e2e/dashboard.spec.ts, which Playwright does collect -- +// and the fix for that would be a suppression list, which is how these guards +// rot. This asks the honest question instead: is any configured runner +// collecting it? +// +// Deliberately .ts/.tsx only. `e2e/fake-backend.test.mjs` is a node:test file +// and no configured runner collects it either, but that is a pre-existing +// finding about a different runner in a different tier, and widening this +// check to .mjs would turn a new gate into a permanently red one on day one. +// See the PR for #3318. +// +// Usage: node scripts/test-discovery-guard.mjs (exit 0 = every test-shaped +// file is collected) +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { tmpdir } from 'node:os' +import { join, relative, resolve, sep } from 'node:path' + +const PACKAGE_ROOT = resolve(import.meta.dirname, '..') + +// Not walked at all. These hold dependencies, build output and reports -- +// nothing a test lives in, and node_modules alone is large enough that walking +// it would dominate the run. +const SKIP_DIRS = new Set([ + '.git', '.nitro', '.output', '.stryker-tmp', '.tanstack', + 'coverage', 'dist', 'node_modules', 'playwright-report', 'public', 'reports', 'test-results', +]) + +// What "looks like a test" means. .ts/.tsx, both suffixes, both the .test and +// the .spec infix -- which is why the canary proof below is a .spec.ts, not a +// .test.ts: the globs under test name both, and a guard that only knew one of +// them would have passed a file placed to dodge it. +const TEST_SHAPED = /(^|[\\/])[^\\/]+\.(test|spec)\.(ts|tsx)$/ + +function toPosix(p) { + return sep === '/' ? p : p.split(sep).join('/') +} + +function relativeToRoot(absolute) { + return toPosix(relative(PACKAGE_ROOT, absolute)) +} + +function walk(dir, found = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue + walk(join(dir, entry.name), found) + } else if (entry.isFile() && TEST_SHAPED.test(entry.name)) { + found.push(relativeToRoot(join(dir, entry.name))) + } + } + return found +} + +function isFile(relPath) { + try { + return statSync(resolve(PACKAGE_ROOT, relPath)).isFile() + } catch { + return false + } +} + +function run(label, cliArgs, parse, env = process.env) { + const result = spawnSync(process.execPath, cliArgs, { + cwd: PACKAGE_ROOT, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + env, + }) + if (result.error) { + console.error(`test-discovery: cannot run the ${label} collector: ${result.error.message}`) + console.error(' run `npm ci` first -- the guard asks the real runners, it does not reimplement them') + process.exit(2) + } + if (result.status !== 0) { + // A collector that failed is not a collector that found nothing. Treating + // it as the latter would make this guard pass on the exact runs where it + // could not see anything. + console.error(`test-discovery: the ${label} collector exited ${result.status}`) + if (result.stderr.trim()) console.error(result.stderr.trim().split('\n').map((l) => ` ${l}`).join('\n')) + process.exit(2) + } + return parse(result.stdout) +} + +// vitest prints one collected file per line, package-relative and posix. Lines +// that are not existing files are dropped: `--list` imports each test module to +// collect it, and a module that prints on import must not be able to inject a +// line that makes an unrun file look collected. +function collectVitest() { + const files = run('vitest', [join(PACKAGE_ROOT, 'node_modules/vitest/vitest.mjs'), 'list', '--filesOnly'], (stdout) => + stdout.split('\n').map((l) => l.trim()).filter(Boolean), + ) + return { runner: 'vitest', command: 'vitest list --filesOnly', files: files.filter(isFile) } +} + +// Playwright's json reporter nests specs under suites; a suite and a spec each +// carry the file they live in, and a spec with no cases in it still has a +// suite entry -- so both are collected, or a file whose only test was skipped +// would look unrun. +// +// The report is read from a FILE, not from stdout, and that is not tidiness. +// Playwright's collection imports every file its testMatch reaches -- including +// e2e/fake-backend.test.mjs, which is a node:test module -- and importing it +// makes node's own runner write a TAP banner to stdout, ahead of the JSON. +// Verified in the node:22-alpine image this gate runs on: `playwright test +// --list --reporter=json 2>&1` opens with "TAP version 13". PLAYWRIGHT_JSON_OUTPUT_NAME +// puts the report in a file, where nothing else can interleave with it. +function collectPlaywright() { + const dir = mkdtempSync(join(tmpdir(), 'test-discovery-')) + const reportPath = join(dir, 'playwright-list.json') + try { + run('playwright', [join(PACKAGE_ROOT, 'node_modules/@playwright/test/cli.js'), 'test', '--list', '--reporter=json'], () => null, { + ...process.env, + PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath, + }) + if (!isFile(reportPath)) { + console.error(`test-discovery: the playwright collector wrote no report at ${reportPath}`) + process.exit(2) + } + let doc + try { + doc = JSON.parse(readFileSync(reportPath, 'utf8')) + } catch (err) { + console.error(`test-discovery: cannot read the playwright collector's report: ${err.message}`) + process.exit(2) + } + const files = new Set() + const walkSuites = (suites) => { + for (const suite of suites ?? []) { + if (suite.file) files.add(toPosix(suite.file)) + for (const spec of suite.specs ?? []) { + if (spec.file) files.add(toPosix(spec.file)) + } + walkSuites(suite.suites) + } + } + walkSuites(doc.suites) + const root = doc.config?.rootDir + const base = root ? relative(PACKAGE_ROOT, resolve(root)) : '' + return { + runner: 'playwright', + command: 'playwright test --list --reporter=json', + files: [...files].map((f) => toPosix(join(base, f))), + } + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} + +const onDisk = walk(PACKAGE_ROOT).sort() +const collectors = [collectVitest(), collectPlaywright()] +const collected = new Set(collectors.flatMap((c) => c.files)) +const unrun = onDisk.filter((f) => !collected.has(f)) + +for (const c of collectors) { + console.log(`test-discovery: ${c.runner} collects ${c.files.length} file(s) (${c.command})`) +} +console.log(`test-discovery: ${onDisk.length} test-shaped .ts/.tsx file(s) under ${relative(PACKAGE_ROOT, PACKAGE_ROOT) || '.'}/`) + +if (unrun.length) { + console.error('') + console.error(`::error::${unrun.length} test file(s) exist that no configured runner collects (#3318):`) + for (const f of unrun) console.error(` - ${f}`) + console.error('') + console.error('vitest collects src/** per its include globs (vitest.config.ts); playwright collects') + console.error('its own testDir (playwright.config.ts, currently e2e/). A test in neither is not') + console.error('running, and a coverage number counts it as though it were. Either:') + console.error(' - move it somewhere the runner that should own it collects from, or') + console.error(' - delete it, if it was never meant to run.') + process.exit(1) +} + +console.log('test-discovery: every test-shaped file is collected. No silently-unrun tests.') diff --git a/arcane/home/honeypot-dashboard/frontend-next/vitest.config.ts b/arcane/home/honeypot-dashboard/frontend-next/vitest.config.ts index cdc01f55..d9bd0a5c 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/vitest.config.ts +++ b/arcane/home/honeypot-dashboard/frontend-next/vitest.config.ts @@ -42,5 +42,46 @@ export default defineConfig({ outputFile: { junit: join(artifactsDir, 'frontend-next-unit-junit.xml') }, } : {}), + // #3318: the tier had no coverage number at all, so nothing here could + // tell a change that deleted tested behaviour from one that did not. + // + // Not `enabled: true` on purpose: coverage instruments every module it + // loads and writes a report tree to disk, and `npm test` is the command + // deploy.yml, the README and a developer's own loop all run. It stays + // uninstrumented and report-free. `npm run test:coverage` -- the one + // command that turns this on -- is what CI and the ratchet use. + // + // `include` is the whole point: the scope is src/ and only src/. Not the + // tests themselves (they are 100% covered by construction and would + // inflate every total), not the configs or scripts, not e2e/, and not + // node_modules. What is left is the code the dashboard actually ships, + // which is the only thing a coverage number should be about. + coverage: { + provider: 'v8', + reportsDirectory: 'coverage', + // json-summary is what scripts/coverage-ratchet.mjs reads; lcovonly is + // what a reviewer loads into an external coverage viewer; text is what + // the CI log shows. `lcov` rather than `lcovonly` would also emit the + // self-contained html report, and that is 5.9MB of per-file assets on a + // run that uploads its coverage to a 7-day artifact -- so it is + // deliberately not the one that generates it. + reporter: ['text', 'json-summary', 'lcovonly'], + include: ['src/**/*.ts', 'src/**/*.tsx'], + exclude: [ + // The tests, the type-only declaration, and the generated route tree + // -- the same three exclusions the test-level `exclude` above already + // reasons about, so "what counts as source" is one list, not two that + // can disagree. Written out rather than inherited: vitest 4 dropped + // its coverage `exclude` defaults (coverageConfigDefaults.exclude is + // now []), so there is nothing to spread. + '**/*.test.ts', + '**/*.test.tsx', + '**/*.spec.ts', + '**/*.spec.tsx', + '**/__tests__/**', + '**/*.d.ts', + 'src/routeTree.gen.ts', + ], + }, }, }) diff --git a/scripts/tests/test_3318_coverage_ratchet.py b/scripts/tests/test_3318_coverage_ratchet.py new file mode 100644 index 00000000..710e0a85 --- /dev/null +++ b/scripts/tests/test_3318_coverage_ratchet.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Pin the #3318 coverage ratchet and test-discovery guard, without a coverage run. + +The issue asks for two proofs -- that the ratchet fails a deliberate drop, and +that the discovery guard catches a canary. Both were demonstrated by hand +before this file existed; a demonstration in a PR description is a transcript, +and a transcript does not stop the next person from widening the tolerance, +loosening a comparison, or quietly deleting the CI step. So each proof is +pinned here as a case that fails if the property regresses. + +Three groups, and why they need different fixtures: + +* `BaselineContract` asserts things about the committed files with no + execution at all -- the baseline's shape, the tolerance, the scope globs, + and the fact that package.json still declares the commands. This is the + anti-weakening half. It is the group that would notice a PR which raised + tolerance.pctPoints to make its own run go green, and it needs neither node + nor an install, so it cannot skip. + +* `RatchetBites` drives the real `coverage-ratchet.mjs` with synthetic + coverage summaries through its `--summary` flag, against the real committed + baseline. No coverage run is needed, and every case is a decision the script + has to get right: the arithmetic, the two independent gates, and the refusal + to pass on a missing measurement or an unusable baseline. Needs node. + +* `DiscoveryGuard` runs the real guard over the real package tree, twice: once + as it stands, and once with a planted canary. Needs node AND an installed + package, because the guard's whole design is to ask the real runners what + they collect rather than reimplement their globs -- with no install there is + nothing to ask, and the guard correctly exits 2 rather than guessing. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +# arcane/home/honeypot-dashboard/ is Arcane's home directory, so `home` is a +# real path segment and not a typo to tidy away. +PACKAGE = ROOT / "arcane" / "home" / "honeypot-dashboard" / "frontend-next" +RATCHET = PACKAGE / "scripts" / "coverage-ratchet.mjs" +GUARD = PACKAGE / "scripts" / "test-discovery-guard.mjs" +BASELINE = PACKAGE / "coverage-baseline.json" +VITEST_CONFIG = PACKAGE / "vitest.config.ts" + +# The canary is planted in the package tree on purpose -- the guard walks it, +# so a canary anywhere else would be testing nothing. .spec.ts rather than +# .test.ts for the reason the guard's own comment gives: vitest's include +# globs only name the .test infix, so this is the shape a test file can take +# and never be collected, and a guard that knew only .test would pass it. +CANARY_REL = "src/lib/__test_3318_discovery_canary.spec.ts" + +# Same skip conditions as test_3315_image_revision.py, for the same reason: the +# scripts/tests lane has no guarantee of an npm install, and a test that fails +# on a missing toolchain is a test that reports the wrong problem. +HAVE_NODE = shutil.which("node") is not None +HAVE_INSTALL = (PACKAGE / "node_modules" / "vitest" / "vitest.mjs").is_file() + + +def run_ratchet(summary: dict | None, cwd: Path = PACKAGE, args: tuple[str, ...] = ()): + """Run the real ratchet, optionally over a synthetic summary. Returns (rc, output).""" + tmp = None + argv = [str(RATCHET), *args] + if summary is not None: + tmp = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) + json.dump(summary, tmp) + tmp.close() + argv += ["--summary", tmp.name] + try: + proc = subprocess.run(["node", *argv], cwd=cwd, capture_output=True, text=True, timeout=120) + return proc.returncode, proc.stdout + proc.stderr + finally: + if tmp is not None: + Path(tmp.name).unlink(missing_ok=True) + + +def summary_from(covered_delta: int = 0, total_delta: int = 0, files: int = 127) -> dict: + """A coverage summary shaped like the real one, moved by the given deltas. + + Shaped rather than hand-written per case so the numbers in each test's + name stay the only place the arithmetic is stated. `files` controls the + per-file entry count, because the ratchet reports a scope change from it. + """ + base = json.loads(BASELINE.read_text()) + totals = {} + for metric, t in base["totals"].items(): + covered = t["covered"] + (covered_delta if metric in ("lines", "branches") else 0) + total = t["total"] + (total_delta if metric in ("lines", "branches") else 0) + totals[metric] = { + "covered": covered, + "total": total, + "skipped": 0, + "pct": round(100 * covered / total, 2) if total else 100, + } + doc = {"total": totals} + for i in range(files): + doc[f"../src/filler{i:03d}.ts"] = totals["lines"] + return doc + + +class BaselineContract(unittest.TestCase): + """What the committed files must keep saying. No execution, so no skip.""" + + def setUp(self): + self.baseline = json.loads(BASELINE.read_text()) + self.package = json.loads((PACKAGE / "package.json").read_text()) + + def test_baseline_is_well_formed(self): + for key in ("about", "recordedAt", "measuredWith", "scope", "tolerance", "files", "totals"): + self.assertIn(key, self.baseline, f"coverage-baseline.json lost {key!r}") + self.assertGreater(self.baseline["files"], 0, "a baseline measuring zero files is not a baseline") + for metric in ("lines", "branches"): + total = self.baseline["totals"][metric] + self.assertGreater(total["total"], 0) + self.assertLessEqual(total["covered"], total["total"]) + # pct is recorded for the human reading the diff; the ratchet + # recomputes it from the counts, so the two must agree here or the + # committed file is describing something the script will not do. + self.assertAlmostEqual(total["pct"], 100 * total["covered"] / total["total"], places=3) + + def test_scope_is_src_only_and_says_so(self): + # The issue's scope, in the file that records the measurement: src/ + # only, and explicitly not tests, not config, not generated. + self.assertIn("src/", self.baseline["scope"]) + for excluded in (".test.", ".spec.", "routeTree.gen.ts"): + self.assertIn(excluded, self.baseline["scope"], f"scope no longer excludes {excluded}") + + def test_tolerance_has_not_been_loosened(self): + tolerance = self.baseline["tolerance"] + # Zero, exactly. This is the gate that fires when a change removes + # tested behaviour, and it is the one a "just this once" PR reaches for + # first. Loosening it is legal only by changing this assertion in the + # same commit, which is the point. + self.assertEqual( + tolerance["coveredCount"], + 0, + "tolerance.coveredCount was raised. It is 0 so that no amount of " + "denominator manipulation can absorb a lost covered line.", + ) + # Bounded below as well as above: a token value like 0.05 would pass a + # <= 1.0 assertion while letting any real regression through. + self.assertGreaterEqual( + tolerance["pctPoints"], + 0.5, + "tolerance.pctPoints was shrunk to a value that cannot catch a regression", + ) + self.assertLessEqual( + tolerance["pctPoints"], + 1.0, + "tolerance.pctPoints was widened beyond the value this issue set", + ) + + def test_commands_and_provider_are_still_declared(self): + scripts = self.package["scripts"] + for name in ("test:coverage", "test:discovery", "coverage:ratchet", "coverage:baseline"): + self.assertIn(name, scripts, f"package.json lost the {name} script") + self.assertIn("@vitest/coverage-v8", self.package["devDependencies"]) + # The rule this lane is held to: no coverage tool beyond v8. + others = [d for d in self.package["devDependencies"] if "coverage" in d and d != "@vitest/coverage-v8"] + self.assertEqual(others, [], f"a second coverage provider appeared: {others}") + + def test_coverage_config_excludes_tests_and_generated(self): + # Asserted as text because that is where the scope actually lives, and + # a config that quietly widened its include globs would move every + # number in the baseline without failing anything else. + config = VITEST_CONFIG.read_text() + self.assertIn("coverage", config) + for glob in ("**/*.test.ts", "**/*.test.tsx", "**/*.spec.ts", "**/*.spec.tsx", "src/routeTree.gen.ts"): + self.assertIn(glob, config, f"vitest.config.ts coverage.exclude lost {glob!r}") + self.assertIn("src/**/*.ts", config, "coverage.include no longer covers src/**/*.ts") + + +@unittest.skipUnless(HAVE_NODE, "node is not on PATH") +class RatchetBites(unittest.TestCase): + """The ratchet's decisions, driven through the real script.""" + + def test_measurement_equal_to_baseline_passes(self): + rc, out = run_ratchet(summary_from()) + self.assertEqual(rc, 0, out) + self.assertIn("no regression", out) + + def test_improvement_passes(self): + # A ratchet that fails on a rise is a ratchet people turn off. + rc, out = run_ratchet(summary_from(covered_delta=40, files=127)) + self.assertEqual(rc, 0, out) + + def test_one_covered_line_lost_fails_even_though_the_percentage_holds(self): + # THE case. One covered line is ~0.0136pp, far inside the 1.0pp + # tolerance, so the percentage gate alone would wave this through -- + # which is exactly why tolerance.coveredCount exists. If this ever + # passes, the ratchet has stopped being a non-regression guard and + # become a rounding check. + rc, out = run_ratchet(summary_from(covered_delta=-1)) + self.assertEqual(rc, 1, out) + self.assertIn("lost 1 covered line", out) + self.assertIn("no tolerance allowed", out) + + def test_one_covered_branch_lost_fails(self): + rc, out = run_ratchet(summary_from(covered_delta=-1)) + self.assertEqual(rc, 1, out) + self.assertIn("lost 1 covered branch", out) + + def test_pct_drop_past_tolerance_fails_with_the_covered_count_intact(self): + # The other direction: a change that ADDS untested source. No covered + # line is lost, so the count gate cannot see it -- the percentage gate + # exists for exactly this. + # + # The delta is derived rather than guessed, because how much new + # uncovered code it takes to move this gate depends on where the + # baseline sits, and at ~11% that is a lot: the denominator has to + # grow past covered/(pct-tolerance) before the percentage moves by + # tolerance at all. A gate that is coarse in one direction and exact in + # the other is the honest shape of a low-coverage ratchet, and + # tolerance.coveredCount is what makes the other direction exact. + base = json.loads(BASELINE.read_text())["totals"]["lines"] + pct = base["pct"] / 100 + tolerance = json.loads(BASELINE.read_text())["tolerance"]["pctPoints"] / 100 + needed = int(base["covered"] / (pct - tolerance)) - base["total"] + 1 + self.assertGreater(needed, 0, "a 1.0pp tolerance should need real new code to trip at this baseline") + rc, out = run_ratchet(summary_from(total_delta=needed)) + self.assertEqual(rc, 1, out) + self.assertIn("fell", out) + self.assertIn("pp, past the", out) + + def test_new_untested_source_inside_tolerance_does_not_fail(self): + # The same arithmetic, the other side: a new route's worth of uncovered + # lines is not a regression and must not turn the gate red. Without + # this, the honest fix for a red ratchet would be to stop running it. + rc, out = run_ratchet(summary_from(total_delta=40)) + self.assertEqual(rc, 0, out) + + def test_small_denominator_change_alone_does_not_fail(self): + # A file leaving src/ moves the denominator. If it was uncovered, the + # percentage goes UP and nothing was lost -- so this passes, and the + # script says the scope moved rather than letting a reader assume the + # number held still for the reason they think. + rc, out = run_ratchet(summary_from(total_delta=-100, files=126)) + self.assertEqual(rc, 0, out) + self.assertIn("scope changed", out) + + def test_missing_summary_fails_loudly_rather_than_passing(self): + rc, out = run_ratchet(None, args=("--summary", "/nonexistent/coverage-summary.json")) + self.assertEqual(rc, 2, out) + self.assertIn("cannot read the coverage summary", out) + self.assertIn("test:coverage", out) + + def test_unusable_baseline_fails_loudly(self): + # A baseline that is not a baseline -- truncated, hand-edited into + # nonsense, or half-merged -- must not be silently replaced or + # silently accepted. Copied into a scratch package root because the + # script resolves the baseline next to itself. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "scripts").mkdir() + shutil.copy(RATCHET, root / "scripts" / RATCHET.name) + (root / "coverage-baseline.json").write_text(json.dumps({"totals": {"lines": {"pct": 60}}})) + summary = root / "summary.json" + summary.write_text(json.dumps(summary_from())) + proc = subprocess.run( + ["node", str(root / "scripts" / RATCHET.name), "--summary", str(summary)], + cwd=root, + capture_output=True, + text=True, + timeout=120, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("not a usable baseline", proc.stdout + proc.stderr) + + def test_update_writes_the_measurement_it_was_given(self): + # --update must still validate the baseline it is about to replace, so + # a regeneration over a broken one fails rather than healing it + # silently. And it must write the measured numbers, not a target. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "scripts").mkdir() + shutil.copy(RATCHET, root / "scripts" / RATCHET.name) + shutil.copy(BASELINE, root / "coverage-baseline.json") + summary = root / "summary.json" + summary.write_text(json.dumps(summary_from(covered_delta=7))) + proc = subprocess.run( + ["node", str(root / "scripts" / RATCHET.name), "--update", "--summary", str(summary)], + cwd=root, + capture_output=True, + text=True, + timeout=120, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + written = json.loads((root / "coverage-baseline.json").read_text()) + self.assertEqual(written["totals"]["lines"]["covered"], json.loads(BASELINE.read_text())["totals"]["lines"]["covered"] + 7) + # The tolerance is not a measurement: regenerating must not move it. + self.assertEqual(written["tolerance"], json.loads(BASELINE.read_text())["tolerance"]) + + +@unittest.skipUnless(HAVE_NODE and HAVE_INSTALL, "node or an npm install of frontend-next is missing") +class DiscoveryGuard(unittest.TestCase): + """The guard over the real tree, and over the real tree plus a canary.""" + + def test_real_tree_is_clean(self): + proc = subprocess.run(["node", str(GUARD)], cwd=PACKAGE, capture_output=True, text=True, timeout=300) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("every test-shaped file is collected", proc.stdout) + # Both runners must have been asked, or the guard is only half a guard. + self.assertIn("vitest collects", proc.stdout) + self.assertIn("playwright collects", proc.stdout) + + def test_canary_is_caught_and_then_removed(self): + canary = PACKAGE / CANARY_REL + self.assertFalse(canary.exists(), f"{CANARY_REL} already exists; refusing to overwrite it") + canary.parent.mkdir(parents=True, exist_ok=True) + canary.write_text("import { expect, it } from 'vitest'\nit('canary', () => expect(1).toBe(1))\n") + self.addCleanup(lambda: canary.unlink(missing_ok=True)) + proc = subprocess.run(["node", str(GUARD)], cwd=PACKAGE, capture_output=True, text=True, timeout=300) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + # The diagnostic has to name the file, or a red run sends the reader + # looking for it. + self.assertIn(CANARY_REL, proc.stdout + proc.stderr) + # And the tree is clean again afterwards -- a guard proof that leaves + # evidence behind is a guard proof the next run trips over. + canary.unlink() + self.assertFalse(canary.exists()) + again = subprocess.run(["node", str(GUARD)], cwd=PACKAGE, capture_output=True, text=True, timeout=300) + self.assertEqual(again.returncode, 0, again.stdout + again.stderr) + + +if __name__ == "__main__": + unittest.main()