From d0cca76d765dc01b888847885bab48622faadaeb Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 12 Oct 2026 08:06:25 +0100 Subject: [PATCH 1/4] ci(mutation): upload the targets' mutation reports A target sits at packages/targets/, one level shallower than the packages in the other layers, so neither upload glob matched it and the command line's and the bundle's HTML reports never reached the artifact. --- .github/workflows/mutation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 153a2cf..5583597 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -69,9 +69,10 @@ jobs: if: always() with: name: mutation-report-shard-${{ matrix.index }} + # A target sits one level shallower than a package in the other layers (packages/targets/ against packages///). path: | packages/*/*/*/reports/mutation/mutation.html - packages/*/reports/mutation/mutation.html + packages/targets/*/reports/mutation/mutation.html if-no-files-found: ignore mutation-result: From 432b302018e0e3402d7b5427b05d35652bd92d3a Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 12 Oct 2026 08:06:44 +0100 Subject: [PATCH 2/4] build(mutation): leave Vanilla Extract style modules out of mutation The Vite plugin's compiler evaluates a .css.ts module to emit its stylesheet and class names, so the instrumented copy never runs in the test process and every mutant in it reports no coverage, which no test can change. Mutating it only pulls the score of every package with styles down. The shard planner's comments no longer claim to mirror the mutate globs exactly, since they leave out more than unit tests. --- .github/scripts/compute-mutation-shards.ts | 4 ++-- stryker.shared.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/compute-mutation-shards.ts b/.github/scripts/compute-mutation-shards.ts index 1b9a8db..22d6618 100644 --- a/.github/scripts/compute-mutation-shards.ts +++ b/.github/scripts/compute-mutation-shards.ts @@ -1,6 +1,6 @@ // Turns the set of packages `_test:mutation` is affected for into a GitHub Actions matrix of weighted shards. Greedy longest-processing-time-first (LPT) bin-packing, not round-robin: a naive `package N -> shard N mod shardCount` assignment can strand the single largest affected package alone on one shard while every other shard finishes and idles waiting on it, since per-package mutation-test cost varies enormously by package size. LPT is a well-known constant-factor approximation of optimal balanced partitioning, the same class of algorithm behind most real-world "spread N differently-sized jobs across K workers evenly" schedulers. // -// The weight proxy is each package's own `src/**/*.ts` line count (excluding `*.test.ts`, mirroring stryker.shared.ts's own default `mutate` exclusions exactly, so the weight actually reflects what Stryker will mutate). This is a proxy for likely mutant count and related-test runtime, not a direct measurement. The only way to know a package's real mutant count ahead of time is Stryker's own dry run, and running that dry run for every affected package just to decide how to shard them would cost as much as the sharding is meant to save. Line count is free and correlates well enough with both to beat round-robin. +// The weight proxy is each package's own `src/**/*.ts` line count (excluding `*.test.ts`, as stryker.shared.ts's own default `mutate` globs do, so the weight approximates what Stryker will mutate; those globs also leave out test support, worker and style modules). This is a proxy for likely mutant count and related-test runtime, not a direct measurement. The only way to know a package's real mutant count ahead of time is Stryker's own dry run, and running that dry run for every affected package just to decide how to shard them would cost as much as the sharding is meant to save. Line count is free and correlates well enough with both to beat round-robin. import { execFileSync } from "node:child_process"; import { appendFileSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -59,7 +59,7 @@ export function affectedMutationPackages( .map((task) => ({ name: task.package, directory: task.directory })); } -/** Every `.ts` file under a package's own `src/`, minus its own unit test files. This is the same scope stryker.shared.ts's default `mutate` glob covers. A package listed by turbo's own dry-run plan is a real workspace package with a real `src/` directory by construction, so a missing `src/` here would be a genuine bug in the affected-package list, not an input this function should tolerate silently. */ +/** Every `.ts` file under a package's own `src/`, minus its own unit test files. This approximates the scope stryker.shared.ts's default `mutate` globs cover. A package listed by turbo's own dry-run plan is a real workspace package with a real `src/` directory by construction, so a missing `src/` here would be a genuine bug in the affected-package list, not an input this function should tolerate silently. */ function packageSourceFiles(directory: string): readonly string[] { const sourceRoot = join(directory, "src"); diff --git a/stryker.shared.ts b/stryker.shared.ts index 67267f0..8050dce 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -31,6 +31,8 @@ export function packageStrykerConfig( "!src/test-support/**", // A worker file a test spawns through `new Worker(new URL(...))` runs in its own thread, outside the sandboxed vitest process Stryker instruments — so its mutants can never be killed, only error or sit uncovered. Mutating test infrastructure no in-process test can observe is noise, not coverage. "!src/**/*.worker.ts", + // A Vanilla Extract style module is evaluated by the Vite plugin's own compiler, which turns it into a stylesheet and the class names a test imports; the instrumented module never runs in the test process, so every one of its mutants reports no coverage, even where a test reads a rule back through `getComputedStyle`. + "!src/**/*.css.ts", ...additionalMutate, ], reporters: ["html", "clear-text", "progress"], From 6da9e139c8241b5bde7689bec0f4944bedc2b883 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 12 Oct 2026 08:07:20 +0100 Subject: [PATCH 3/4] build(graph-web): mutation test the web app Its unit tests render the screens in happy-dom with the Vanilla Extract plugin, as the UI packages' do, and those packages already run under Stryker, so nothing kept the app out of mutation testing. Its store worker and style modules stay out through the shared mutate globs. --- packages/targets/graph-web/package.json | 5 +++++ packages/targets/graph-web/stryker.config.ts | 5 +++++ pnpm-lock.yaml | 9 +++++++++ 3 files changed, 19 insertions(+) create mode 100644 packages/targets/graph-web/stryker.config.ts diff --git a/packages/targets/graph-web/package.json b/packages/targets/graph-web/package.json index 67df84a..430cba8 100644 --- a/packages/targets/graph-web/package.json +++ b/packages/targets/graph-web/package.json @@ -30,6 +30,8 @@ "_test:coverage": "vitest run --project unit --coverage", "test:smoke": "turbo run _test:smoke", "_test:smoke": "vitest run --project smoke", + "test:mutation": "turbo run _test:mutation", + "_test:mutation": "stryker run stryker.config.ts", "test:e2e": "turbo run _test:e2e", "_test:e2e": "playwright test" }, @@ -63,6 +65,9 @@ "@exadev/process-graph-cli": "workspace:*", "@playwright/test": "catalog:", "@resvg/resvg-js": "catalog:", + "@stryker-mutator/core": "catalog:", + "@stryker-mutator/typescript-checker": "catalog:", + "@stryker-mutator/vitest-runner": "catalog:", "@testing-library/dom": "catalog:", "@testing-library/react": "catalog:", "@types/node": "catalog:", diff --git a/packages/targets/graph-web/stryker.config.ts b/packages/targets/graph-web/stryker.config.ts new file mode 100644 index 0000000..9ee47bc --- /dev/null +++ b/packages/targets/graph-web/stryker.config.ts @@ -0,0 +1,5 @@ +import { packageStrykerConfig } from "../../../stryker.shared.ts"; + +export default packageStrykerConfig({ + additionalMutate: ["src/**/*.tsx", "!src/**/*.test.tsx"], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1aaeffe..77a6d37 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1233,6 +1233,15 @@ importers: '@resvg/resvg-js': specifier: 'catalog:' version: 2.6.2 + '@stryker-mutator/core': + specifier: 'catalog:' + version: 10.0.0(@types/node@26.6.4) + '@stryker-mutator/typescript-checker': + specifier: 'catalog:' + version: 10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(typescript@6.0.3) + '@stryker-mutator/vitest-runner': + specifier: 'catalog:' + version: 10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(vitest@4.1.11) '@testing-library/dom': specifier: 'catalog:' version: 10.4.2 From 4152332b5125d73e9c762e0aeb979d722beaab4c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 12 Oct 2026 08:07:40 +0100 Subject: [PATCH 4/4] build(mutation): derive a break threshold from a measured baseline breakThresholdFromBaseline takes a completed run's killed, timed-out, survived and no-coverage counts and returns the score with every timeout counted as survived, floored to a whole percentage. A timeout depends on runner load and the other verdicts do not, so this is the lowest score the same code and tests can produce, and the gate needs no hand-picked margin. The shared config also names the package left out of mutation testing and why. --- stryker.shared.ts | 40 ++++++++++++++++- stryker.shared.unit.test.ts | 87 +++++++++++++++++++++++++++++++++++++ tsconfig.json | 1 + vitest.config.ts | 1 + 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 stryker.shared.unit.test.ts diff --git a/stryker.shared.ts b/stryker.shared.ts index 8050dce..a55d547 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -1,12 +1,50 @@ import type { PartialStrykerOptions } from "@stryker-mutator/api/core"; +/** + * Packages with no Stryker run, each for a reason a run cannot get round: + * + * - `@exadev/process-graph-sqlite-wasm`: its tests drive a real browser over OPFS, which Stryker's sandboxed copies cannot start, so a run could only score the few in-process unit tests as if they were the package's coverage. + * + * Every other workspace package has its own `stryker.config.ts` and `_test:mutation` script. + */ export interface PackageStrykerOptions { - /** Every package's own `stryker.config.ts` passes this. Left unset (`undefined`) until a real CI run has actually measured a baseline. A mutation-score gate chosen before ever running the suite is exactly the arbitrary magic number this workspace's own conventions exist to avoid. */ + /** Every package's own `stryker.config.ts` passes this, as `breakThresholdFromBaseline` of a completed CI run's measured counts. Left unset (`undefined`) until a real CI run has measured a baseline: a mutation-score gate chosen before ever running the suite is exactly the arbitrary magic number this workspace's own conventions exist to avoid. */ readonly breakThreshold?: number; /** Extra glob patterns to mutate, appended to the package's own `src/**\/*.ts` default. */ readonly additionalMutate?: readonly string[]; } +/** One package's verdict counts from a completed Stryker run, as its clear-text report's `All files` row prints them (`# killed`, `# timeout`, `# survived`, `# no cov`). Errored and ignored mutants are left out, as Stryker leaves them out of the score. */ +export interface MutationBaseline { + readonly killed: number; + readonly timedOut: number; + readonly survived: number; + readonly noCoverage: number; +} + +/** Stryker states a mutation score, and compares it with `thresholds.break`, as a percentage. */ +const PERCENT = 100; + +/** + * The break threshold a measured baseline supports: the package's mutation score with every timed-out mutant counted as survived, floored to a whole percentage. + * + * Stryker counts a timed-out mutant as detected, but whether a mutant runs past its time budget depends on how loaded the runner is, so the same mutant can time out on one run and complete, killed or survived, on the next. Killed, survived and no-coverage verdicts depend only on the code and its tests. Counting every timeout as survived therefore gives the lowest score the same code and tests can produce, so the gate absorbs run-to-run variation without a margin picked by hand, and fails only when a change loses a kill. Flooring keeps the threshold a whole number at or below that score. + * + * The result is never below zero and never above the measured score (killed plus timed out over every scored mutant). Raise a package's threshold by re-measuring it once its tests kill more mutants. + * @throws RangeError when the baseline scored no mutant, since a score over nothing measures nothing. + */ +export function breakThresholdFromBaseline(baseline: MutationBaseline): number { + const { killed, timedOut, survived, noCoverage } = baseline; + const scored = killed + timedOut + survived + noCoverage; + if (scored === 0) { + throw new RangeError( + "a mutation baseline needs at least one scored mutant to derive a break threshold from", + ); + } + + return Math.floor((killed * PERCENT) / scored); +} + /** * Deliberately no `incremental`/`incrementalFile`: `@stryker-mutator/vitest-runner`'s own `capabilities()` reports no test-location support, so per StrykerJS's own documented incremental-mode limitations ("test runner plugins offer varying levels of support... 'Nothing' means only mutant changes are detected, not test changes"), Stryker has no way to notice a test file changed and will silently reuse a stale cached verdict for every mutant that test used to cover. Confirmed directly: a package's own mutation score stayed at its pre-fix value in CI, reusing a restored incremental report, even after new tests were added specifically to kill previously-surviving mutants in that exact package. */ diff --git a/stryker.shared.unit.test.ts b/stryker.shared.unit.test.ts new file mode 100644 index 0000000..b60a425 --- /dev/null +++ b/stryker.shared.unit.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; + +import { + breakThresholdFromBaseline, + packageStrykerConfig, +} from "./stryker.shared.ts"; + +describe("breakThresholdFromBaseline", () => { + it.each([ + { + name: "counts every timed-out mutant as survived", + baseline: { killed: 6, timedOut: 2, survived: 2, noCoverage: 0 }, + threshold: 60, + }, + { + name: "counts mutants with no coverage against the score", + baseline: { killed: 3, timedOut: 0, survived: 0, noCoverage: 1 }, + threshold: 75, + }, + { + name: "floors a fractional score to the whole percentage below it", + baseline: { killed: 2, timedOut: 0, survived: 1, noCoverage: 0 }, + threshold: 66, + }, + { + name: "is the whole score when every mutant is killed", + baseline: { killed: 7, timedOut: 0, survived: 0, noCoverage: 0 }, + threshold: 100, + }, + { + name: "is zero when no mutant is killed, however many timed out", + baseline: { killed: 0, timedOut: 4, survived: 1, noCoverage: 1 }, + threshold: 0, + }, + ])("$name", ({ baseline, threshold }) => { + expect(breakThresholdFromBaseline(baseline)).toBe(threshold); + }); + + it("never exceeds the measured score, which counts timeouts as detected", () => { + const baseline = { killed: 5, timedOut: 3, survived: 1, noCoverage: 2 }; + const scored = + baseline.killed + + baseline.timedOut + + baseline.survived + + baseline.noCoverage; + const detected = baseline.killed + baseline.timedOut; + const threshold = breakThresholdFromBaseline(baseline); + + // Scaled by the all-killed threshold, which is the whole percentage, so the test names no percentage scale of its own. + expect(threshold * scored).toBeLessThanOrEqual( + breakThresholdFromBaseline({ + killed: 1, + timedOut: 0, + survived: 0, + noCoverage: 0, + }) * detected, + ); + }); + + it("rejects a baseline that scored no mutant", () => { + expect(() => + breakThresholdFromBaseline({ + killed: 0, + timedOut: 0, + survived: 0, + noCoverage: 0, + }), + ).toThrow(RangeError); + }); +}); + +describe("packageStrykerConfig", () => { + it("gates on the break threshold it is given", () => { + const baseline = { killed: 3, timedOut: 0, survived: 0, noCoverage: 1 }; + const breakThreshold = breakThresholdFromBaseline(baseline); + + expect(packageStrykerConfig({ breakThreshold }).thresholds).toEqual({ + high: 90, + low: 70, + break: breakThreshold, + }); + }); + + it("sets no break threshold until a baseline is measured", () => { + expect(packageStrykerConfig().thresholds).toEqual({ high: 90, low: 70 }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 6bf41e2..6f9ae2b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,6 +11,7 @@ "release-workspace.config.unit.test.ts", "eslint.shared.ts", "stryker.shared.ts", + "stryker.shared.unit.test.ts", ".github/scripts/**/*.ts", "scripts/**/*.ts" ] diff --git a/vitest.config.ts b/vitest.config.ts index 54143c8..dddffc5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ ".github/scripts/**/*.test.ts", "scripts/**/*.test.ts", "release-workspace.config.unit.test.ts", + "stryker.shared.unit.test.ts", ], }, },