build(mutation): gate each package on a break threshold derived from its baseline

.github/scripts/compute-mutation-shards.ts

@@ -1,6 +1,6 @@
 // Turns the set of packages `_test:mutation` is affected for into a GitHub Actions matrix of weighted shards. Greedy longest-processing-time-first (LPT) bin-packing, not round-robin: a naive `package N -> shard N mod shardCount` assignment can strand the single largest affected package alone on one shard while every other shard finishes and idles waiting on it, since per-package mutation-test cost varies enormously by package size. LPT is a well-known constant-factor approximation of optimal balanced partitioning, the same class of algorithm behind most real-world "spread N differently-sized jobs across K workers evenly" schedulers.
 //
-// The weight proxy is each package's own `src/**/*.ts` line count (excluding `*.test.ts`, mirroring stryker.shared.ts's own default `mutate` exclusions exactly, so the weight actually reflects what Stryker will mutate). This is a proxy for likely mutant count and related-test runtime, not a direct measurement. The only way to know a package's real mutant count ahead of time is Stryker's own dry run, and running that dry run for every affected package just to decide how to shard them would cost as much as the sharding is meant to save. Line count is free and correlates well enough with both to beat round-robin.
+// The weight proxy is each package's own `src/**/*.ts` line count (excluding `*.test.ts`, as stryker.shared.ts's own default `mutate` globs do, so the weight approximates what Stryker will mutate; those globs also leave out test support, worker and style modules). This is a proxy for likely mutant count and related-test runtime, not a direct measurement. The only way to know a package's real mutant count ahead of time is Stryker's own dry run, and running that dry run for every affected package just to decide how to shard them would cost as much as the sharding is meant to save. Line count is free and correlates well enough with both to beat round-robin.
 import { execFileSync } from "node:child_process";
 import { appendFileSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
@@ -59,7 +59,7 @@ export function affectedMutationPackages(
     .map((task) => ({ name: task.package, directory: task.directory }));
 }
 
-/** Every `.ts` file under a package's own `src/`, minus its own unit test files. This is the same scope stryker.shared.ts's default `mutate` glob covers. A package listed by turbo's own dry-run plan is a real workspace package with a real `src/` directory by construction, so a missing `src/` here would be a genuine bug in the affected-package list, not an input this function should tolerate silently. */
+/** Every `.ts` file under a package's own `src/`, minus its own unit test files. This approximates the scope stryker.shared.ts's default `mutate` globs cover. A package listed by turbo's own dry-run plan is a real workspace package with a real `src/` directory by construction, so a missing `src/` here would be a genuine bug in the affected-package list, not an input this function should tolerate silently. */
 function packageSourceFiles(directory: string): readonly string[] {
   const sourceRoot = join(directory, "src");
 

.github/workflows/mutation.yml

@@ -69,9 +69,10 @@ jobs:
         if: always()
         with:
           name: mutation-report-shard-${{ matrix.index }}
+          # A target sits one level shallower than a package in the other layers (packages/targets/<package> against packages/<layer>/<domain>/<package>).
           path: |
             packages/*/*/*/reports/mutation/mutation.html
-            packages/*/reports/mutation/mutation.html
+            packages/targets/*/reports/mutation/mutation.html
           if-no-files-found: ignore
 
   mutation-result:

packages/targets/graph-web/package.json

@@ -30,6 +30,8 @@
     "_test:coverage": "vitest run --project unit --coverage",
     "test:smoke": "turbo run _test:smoke",
     "_test:smoke": "vitest run --project smoke",
+    "test:mutation": "turbo run _test:mutation",
+    "_test:mutation": "stryker run stryker.config.ts",
     "test:e2e": "turbo run _test:e2e",
     "_test:e2e": "playwright test"
   },
@@ -63,6 +65,9 @@
     "@exadev/process-graph-cli": "workspace:*",
     "@playwright/test": "catalog:",
     "@resvg/resvg-js": "catalog:",
+    "@stryker-mutator/core": "catalog:",
+    "@stryker-mutator/typescript-checker": "catalog:",
+    "@stryker-mutator/vitest-runner": "catalog:",
     "@testing-library/dom": "catalog:",
     "@testing-library/react": "catalog:",
     "@types/node": "catalog:",

packages/targets/graph-web/stryker.config.ts

@@ -0,0 +1,5 @@
+import { packageStrykerConfig } from "../../../stryker.shared.ts";
+
+export default packageStrykerConfig({
+  additionalMutate: ["src/**/*.tsx", "!src/**/*.test.tsx"],
+});

stryker.shared.ts

@@ -1,12 +1,50 @@
 import type { PartialStrykerOptions } from "@stryker-mutator/api/core";
 
+/**
+ * Packages with no Stryker run, each for a reason a run cannot get round:
+ *
+ * - `@exadev/process-graph-sqlite-wasm`: its tests drive a real browser over OPFS, which Stryker's sandboxed copies cannot start, so a run could only score the few in-process unit tests as if they were the package's coverage.
+ *
+ * Every other workspace package has its own `stryker.config.ts` and `_test:mutation` script.
+ */
 export interface PackageStrykerOptions {
-  /** Every package's own `stryker.config.ts` passes this. Left unset (`undefined`) until a real CI run has actually measured a baseline. A mutation-score gate chosen before ever running the suite is exactly the arbitrary magic number this workspace's own conventions exist to avoid. */
+  /** Every package's own `stryker.config.ts` passes this, as `breakThresholdFromBaseline` of a completed CI run's measured counts. Left unset (`undefined`) until a real CI run has measured a baseline: a mutation-score gate chosen before ever running the suite is exactly the arbitrary magic number this workspace's own conventions exist to avoid. */
   readonly breakThreshold?: number;
   /** Extra glob patterns to mutate, appended to the package's own `src/**\/*.ts` default. */
   readonly additionalMutate?: readonly string[];
 }
 
+/** One package's verdict counts from a completed Stryker run, as its clear-text report's `All files` row prints them (`# killed`, `# timeout`, `# survived`, `# no cov`). Errored and ignored mutants are left out, as Stryker leaves them out of the score. */
+export interface MutationBaseline {
+  readonly killed: number;
+  readonly timedOut: number;
+  readonly survived: number;
+  readonly noCoverage: number;
+}
+
+/** Stryker states a mutation score, and compares it with `thresholds.break`, as a percentage. */
+const PERCENT = 100;
+
+/**
+ * The break threshold a measured baseline supports: the package's mutation score with every timed-out mutant counted as survived, floored to a whole percentage.
+ *
+ * Stryker counts a timed-out mutant as detected, but whether a mutant runs past its time budget depends on how loaded the runner is, so the same mutant can time out on one run and complete, killed or survived, on the next. Killed, survived and no-coverage verdicts depend only on the code and its tests. Counting every timeout as survived therefore gives the lowest score the same code and tests can produce, so the gate absorbs run-to-run variation without a margin picked by hand, and fails only when a change loses a kill. Flooring keeps the threshold a whole number at or below that score.
+ *
+ * The result is never below zero and never above the measured score (killed plus timed out over every scored mutant). Raise a package's threshold by re-measuring it once its tests kill more mutants.
+ * @throws RangeError when the baseline scored no mutant, since a score over nothing measures nothing.
+ */
+export function breakThresholdFromBaseline(baseline: MutationBaseline): number {
+  const { killed, timedOut, survived, noCoverage } = baseline;
+  const scored = killed + timedOut + survived + noCoverage;
+  if (scored === 0) {
+    throw new RangeError(
+      "a mutation baseline needs at least one scored mutant to derive a break threshold from",
+    );
+  }
+
+  return Math.floor((killed * PERCENT) / scored);
+}
+
 /**
  * Deliberately no `incremental`/`incrementalFile`: `@stryker-mutator/vitest-runner`'s own `capabilities()` reports no test-location support, so per StrykerJS's own documented incremental-mode limitations ("test runner plugins offer varying levels of support... 'Nothing' means only mutant changes are detected, not test changes"), Stryker has no way to notice a test file changed and will silently reuse a stale cached verdict for every mutant that test used to cover. Confirmed directly: a package's own mutation score stayed at its pre-fix value in CI, reusing a restored incremental report, even after new tests were added specifically to kill previously-surviving mutants in that exact package.
  */
@@ -31,6 +69,8 @@ export function packageStrykerConfig(
       "!src/test-support/**",
       // A worker file a test spawns through `new Worker(new URL(...))` runs in its own thread, outside the sandboxed vitest process Stryker instruments — so its mutants can never be killed, only error or sit uncovered. Mutating test infrastructure no in-process test can observe is noise, not coverage.
       "!src/**/*.worker.ts",
+      // A Vanilla Extract style module is evaluated by the Vite plugin's own compiler, which turns it into a stylesheet and the class names a test imports; the instrumented module never runs in the test process, so every one of its mutants reports no coverage, even where a test reads a rule back through `getComputedStyle`.
+      "!src/**/*.css.ts",
       ...additionalMutate,
     ],
     reporters: ["html", "clear-text", "progress"],

stryker.shared.unit.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  breakThresholdFromBaseline,
+  packageStrykerConfig,
+} from "./stryker.shared.ts";
+
+describe("breakThresholdFromBaseline", () => {
+  it.each([
+    {
+      name: "counts every timed-out mutant as survived",
+      baseline: { killed: 6, timedOut: 2, survived: 2, noCoverage: 0 },
+      threshold: 60,
+    },
+    {
+      name: "counts mutants with no coverage against the score",
+      baseline: { killed: 3, timedOut: 0, survived: 0, noCoverage: 1 },
+      threshold: 75,
+    },
+    {
+      name: "floors a fractional score to the whole percentage below it",
+      baseline: { killed: 2, timedOut: 0, survived: 1, noCoverage: 0 },
+      threshold: 66,
+    },
+    {
+      name: "is the whole score when every mutant is killed",
+      baseline: { killed: 7, timedOut: 0, survived: 0, noCoverage: 0 },
+      threshold: 100,
+    },
+    {
+      name: "is zero when no mutant is killed, however many timed out",
+      baseline: { killed: 0, timedOut: 4, survived: 1, noCoverage: 1 },
+      threshold: 0,
+    },
+  ])("$name", ({ baseline, threshold }) => {
+    expect(breakThresholdFromBaseline(baseline)).toBe(threshold);
+  });
+
+  it("never exceeds the measured score, which counts timeouts as detected", () => {
+    const baseline = { killed: 5, timedOut: 3, survived: 1, noCoverage: 2 };
+    const scored =
+      baseline.killed +
+      baseline.timedOut +
+      baseline.survived +
+      baseline.noCoverage;
+    const detected = baseline.killed + baseline.timedOut;
+    const threshold = breakThresholdFromBaseline(baseline);
+
+    // Scaled by the all-killed threshold, which is the whole percentage, so the test names no percentage scale of its own.
+    expect(threshold * scored).toBeLessThanOrEqual(
+      breakThresholdFromBaseline({
+        killed: 1,
+        timedOut: 0,
+        survived: 0,
+        noCoverage: 0,
+      }) * detected,
+    );
+  });
+
+  it("rejects a baseline that scored no mutant", () => {
+    expect(() =>
+      breakThresholdFromBaseline({
+        killed: 0,
+        timedOut: 0,
+        survived: 0,
+        noCoverage: 0,
+      }),
+    ).toThrow(RangeError);
+  });
+});
+
+describe("packageStrykerConfig", () => {
+  it("gates on the break threshold it is given", () => {
+    const baseline = { killed: 3, timedOut: 0, survived: 0, noCoverage: 1 };
+    const breakThreshold = breakThresholdFromBaseline(baseline);
+
+    expect(packageStrykerConfig({ breakThreshold }).thresholds).toEqual({
+      high: 90,
+      low: 70,
+      break: breakThreshold,
+    });
+  });
+
+  it("sets no break threshold until a baseline is measured", () => {
+    expect(packageStrykerConfig().thresholds).toEqual({ high: 90, low: 70 });
+  });
+});

tsconfig.json

@@ -11,6 +11,7 @@
     "release-workspace.config.unit.test.ts",
     "eslint.shared.ts",
     "stryker.shared.ts",
+    "stryker.shared.unit.test.ts",
     ".github/scripts/**/*.ts",
     "scripts/**/*.ts"
   ]

vitest.config.ts

@@ -11,6 +11,7 @@ export default defineConfig({
             ".github/scripts/**/*.test.ts",
             "scripts/**/*.test.ts",
             "release-workspace.config.unit.test.ts",
+            "stryker.shared.unit.test.ts",
           ],
         },
       },