From 2a8db289c59e14041859294a1df34defe5fa4a70 Mon Sep 17 00:00:00 2001 From: choskeli <98036866+choskeli@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:11:37 -0400 Subject: [PATCH] ci: distribute fresh mutations across verified native ranges --- .github/workflows/mutation.yml | 2 - docs/mutation-cache.md | 24 ++- package-lock.json | 1 + package.json | 1 + scripts/full-mutations.integration.test.mjs | 28 +++- scripts/full-mutations.mjs | 19 +-- scripts/full-mutations.test.mjs | 14 -- scripts/mutation-partitions.mjs | 107 ++++++++++++++ scripts/mutation-partitions.test.mjs | 156 ++++++++++++++++++++ scripts/mutation-policy-inputs.mjs | 30 ++++ scripts/mutation-policy.mjs | 17 ++- scripts/mutation-shards-policy.test.mjs | 68 ++++++++- scripts/mutation-workflow.test.mjs | 6 +- 13 files changed, 424 insertions(+), 49 deletions(-) create mode 100644 scripts/mutation-partitions.mjs create mode 100644 scripts/mutation-partitions.test.mjs diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 6acb838..c41b54c 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -80,8 +80,6 @@ jobs: ref: ${{ github.sha }} persist-credentials: false - uses: ./.github/actions/setup-project - with: - install: 'false' - id: plan run: | matrix=$(node scripts/full-mutations.mjs plan) diff --git a/docs/mutation-cache.md b/docs/mutation-cache.md index cc9b459..44809ad 100644 --- a/docs/mutation-cache.md +++ b/docs/mutation-cache.md @@ -18,8 +18,10 @@ current discovery. `npm run mutation:full` remains a single fresh local campaign Scheduled CI, full manual CI and exact-source release validation instead use the native GitHub matrix in `.github/workflows/mutation.yml`. Node's filesystem glob -discovers every parser source; sorted whole files are assigned round-robin to at -most 11 nonempty parser jobs. The existing pure-runtime campaign occupies the +discovers every parser source. The pinned Stryker instrumenter generates the +complete current mutant inventory without running tests. A small planner joins +overlapping mutant spans into indivisible components and assigns larger components +first to the least-populated of at most 11 nonempty parser jobs. The existing pure-runtime campaign occupies the twelfth mutation slot. This ceiling is not a measured optimum or a cap on other CI jobs. Each parser job retains two Stryker workers, the ordinary full Vitest suite, native parser-test discovery and a fresh Stryker coverage dry run. @@ -31,8 +33,12 @@ event SHA. Caller inputs cannot select other code within the event's cache scope Fresh jobs remove native working state before `--force --mutate`, because force alone can retain out-of-scope results from an old incremental report. They never -restore or publish mutation-result caches. The collector checks every assigned -file exactly once, current source/test bytes, common configuration and declared +restore or publish mutation-result caches. Stryker has no distributed mutant-ID +selector: jobs use its native line/column ranges. A range must fully contain a +mutated AST node, so arbitrary adjacent line splits are unsafe. The planner +re-instruments each assignment and requires its exact mutant identities to match +the complete inventory, with no missing, extra or duplicate ownership. The +collector repeats this check against the unchanged raw reports, current source/test bytes, common configuration and declared inputs, clean process completion, run/attempt/commit identity and artifact/report digests. It applies the existing source-bound equivalent proofs and 100% adjusted policy across unchanged raw reports, resolving test IDs within each report. @@ -52,9 +58,15 @@ native all-skipped suites (such as an absent optional external sample) without a filename exception; every runnable suite must appear consistently in every shard. Ordinary PR/main runs, including cache misses, retain the single-runner path. -File workloads may be uneven; runner count alone does not establish a latency -guarantee. The short native regression in `scripts/full-mutations.integration.test.mjs` +Mutant counts are only a scheduling heuristic: individual tests and timeouts vary +widely, and a method-spanning mutant prevents further safe subdivision. The +planner uses no historical timing store, result cache, dependency graph or report +rewrite. Planning now installs the locked native instrumenter; each shard still +pays for its own ordinary suite and coverage dry run. The short native regression in `scripts/full-mutations.integration.test.mjs` checks the real runner/collector, fresh state, changed inputs and failed evidence. +It also verifies a file split across jobs and rejects a missing mutant even when +the altered report's digest is updated. Ordinary incremental execution remains +whole-scope. See the pinned native [range containment implementation](https://github.com/stryker-mutator/stryker-js/blob/959127763e5bbe2eb65f5341a8161780d6d8a476/packages/instrumenter/src/transformers/babel-transformer.ts). Native reuse includes killed, timeout and reviewed-equivalent outcomes. It is not fresh execution: reused native records omit `duration`. The audit permits an diff --git a/package-lock.json b/package-lock.json index f7e706b..808bb7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@hyperjump/json-schema-formats": "1.1.0", "@jazzer.js/core": "4.0.0", "@stryker-mutator/core": "10.1.1", + "@stryker-mutator/instrumenter": "10.1.1", "@stryker-mutator/vitest-runner": "10.1.1", "@types/node": "24.19.2", "@typescript/native": "npm:typescript@7.0.2", diff --git a/package.json b/package.json index b342749..7dbd0c9 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "@jazzer.js/core": "4.0.0", "@typescript/native": "npm:typescript@7.0.2", "@stryker-mutator/core": "10.1.1", + "@stryker-mutator/instrumenter": "10.1.1", "eslint-plugin-security": "4.2.0", "globals": "17.13.0", "@stryker-mutator/vitest-runner": "10.1.1", diff --git a/scripts/full-mutations.integration.test.mjs b/scripts/full-mutations.integration.test.mjs index d32c5ae..c02c85a 100644 --- a/scripts/full-mutations.integration.test.mjs +++ b/scripts/full-mutations.integration.test.mjs @@ -4,7 +4,7 @@ import { cp, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/pr import { join } from 'node:path'; import { test } from 'node:test'; import { currentSources, runShard, collectShards } from './full-mutations.mjs'; -import { sourceContext } from './mutation-policy.mjs'; +import { sha256, sourceContext } from './mutation-policy.mjs'; import { ENCODER_TEST_NAMES } from './verify-encoder-discovery.mjs'; import { repositoryRoot } from './release-utils.mjs'; @@ -18,8 +18,8 @@ async function fixture() { const files = { '.gitignore': 'node_modules/\n.cache/\nreports/\n.stryker-tmp/\n', 'package.json': '{"type":"module"}', - 'src/mp3/a.ts': 'export const positive = (n: number) => n > 0;\n', - 'src/mp3/b.ts': 'export const negative = (n: number) => n < 0;\n', + 'src/mp3/a.ts': + 'export const positive = (n: number) => n > 0;\nexport const negative = (n: number) => n < 0;\n', 'vitest.config.ts': "export default { test: { include: ['tests/**/*.test.ts'], maxWorkers: 1 } };", 'vitest.mutation.config.ts': @@ -27,8 +27,7 @@ async function fixture() { 'stryker.config.json': JSON.stringify(config), 'tests/mp3/optional.test.ts': "import {it} from 'vitest'; it.skip('optional sample',()=>{});", 'tests/mp3/free-format-real.test.ts': `import {test,expect} from 'vitest'; - import {positive} from '../../src/mp3/a'; - import {negative} from '../../src/mp3/b'; + import {positive,negative} from '../../src/mp3/a'; for(const name of ${JSON.stringify(ENCODER_TEST_NAMES)}) test(name,()=>{ for(const n of [-1,0,1]) { expect(positive(n)).toBe(n===1); @@ -68,7 +67,7 @@ async function fixture() { } test( - 'native whole-file shards remain fresh and collection fails closed', + 'native complete ranges split one file, remain fresh and collection fails closed', { timeout: 90000 }, async () => { const root = await fixture(); @@ -90,10 +89,27 @@ test( assert.equal(summary.adjustedScore, 100); assert.equal(summary.execution.reused, 0); assert.equal(summary.execution.shards, 2); + assert.equal(summary.perFile['src/mp3/a.ts'].total, 10); const native = JSON.parse( await readFile(join(directory, `full-parser-shard-${attempt}-0/mutation.json`), 'utf8'), ); assert.equal(Object.hasOwn(native.testFiles, 'tests/mp3/optional.test.ts'), false); + const shardDirectory = join(directory, `full-parser-shard-${attempt}-0`); + const reportPath = join(shardDirectory, 'mutation.json'); + const manifestPath = join(shardDirectory, 'complete.json'); + const reportBytes = await readFile(reportPath); + const manifestBytes = await readFile(manifestPath); + const manifest = JSON.parse(manifestBytes); + native.files['src/mp3/a.ts'].mutants.pop(); + const incomplete = JSON.stringify(native); + await writeFile(reportPath, incomplete); + await writeFile( + manifestPath, + JSON.stringify({ ...manifest, reportSha256: sha256(incomplete) }), + ); + await assert.rejects(collectShards(root, directory), /native partition mutant/); + await writeFile(reportPath, reportBytes); + await writeFile(manifestPath, manifestBytes); await assert.rejects(readFile(join(root, '.cache/mutation/native-baseline.json')), { code: 'ENOENT', }); diff --git a/scripts/full-mutations.mjs b/scripts/full-mutations.mjs index 48f5312..4df8562 100644 --- a/scripts/full-mutations.mjs +++ b/scripts/full-mutations.mjs @@ -9,17 +9,11 @@ import { mutationEnvironment, mutationInputs, fingerprint } from './mutation-cac import { runnableParserTestFiles, verifyParserTestDiscovery } from './parser-test-discovery.mjs'; import { assertEncoderShardDiscovery } from './verify-encoder-discovery.mjs'; import { runMutationProcess } from './mutation-process.mjs'; +import { mutationPartitions } from './mutation-partitions.mjs'; import { isMain, readJson, repositoryRoot } from './release-utils.mjs'; // Reserve the twelfth mutation runner for the independent pure-runtime campaign. export const MAX_PARSER_SHARDS = 11; -export function partitionSources(paths) { - assert.ok(paths.length > 0, 'Empty parser source inventory'); - assert.equal(new Set(paths).size, paths.length, 'Duplicate parser source'); - const groups = Array.from({ length: Math.min(MAX_PARSER_SHARDS, paths.length) }, () => []); - paths.toSorted().forEach((file, index) => groups[index % groups.length].push(file)); - return groups; -} export async function currentSources(root) { const paths = (await Array.fromAsync(glob(PARSER_GLOB, { cwd: root }))) @@ -67,7 +61,7 @@ async function freshTests(root, scope, env, log) { export async function runShard(root, shard) { const sources = await currentSources(root); - const scopes = partitionSources([...sources.keys()]); + const { scopes } = await mutationPartitions(sources, MAX_PARSER_SHARDS); assert.ok(Number.isInteger(shard) && scopes[shard], 'Invalid parser shard'); const output = join(root, 'reports/mutation'); const lock = join(root, '.cache/mutation/run.lock'); @@ -179,7 +173,7 @@ export async function collectShards(root, directory) { const output = join(root, 'reports/mutation'); await rm(output, { recursive: true, force: true }); const sources = await currentSources(root); - const scopes = partitionSources([...sources.keys()]); + const { scopes, inventories } = await mutationPartitions(sources, MAX_PARSER_SHARDS); const expected = invocation(root); const names = scopes.map((_, index) => artifactName(expected.attempt, index)); assert.deepEqual( @@ -196,7 +190,7 @@ export async function collectShards(root, directory) { const inputs = await mutationInputs(root, mutationEnvironment()); const reports = validateShardEvidence(entries, scopes, expected, inputs); const policy = await readJson(join(root, 'mutation-equivalents.json')); - const summary = auditMutationShards(reports, policy, sources, scopes); + const summary = auditMutationShards(reports, policy, sources, scopes, inventories); await freshTestSources(root, reports); assertEncoderShardDiscovery(reports); assert.equal(fingerprint(await mutationInputs(root, mutationEnvironment())), fingerprint(inputs)); @@ -221,7 +215,10 @@ if (isMain(import.meta.url)) { const [command, argument, ...extra] = process.argv.slice(2); assert.equal(extra.length, 0, 'Unexpected arguments'); if (command === 'plan' && argument === undefined) { - const scopes = partitionSources([...(await currentSources(repositoryRoot)).keys()]); + const { scopes } = await mutationPartitions( + await currentSources(repositoryRoot), + MAX_PARSER_SHARDS, + ); console.log(JSON.stringify({ include: scopes.map((_, shard) => ({ shard })) })); } else if (command === 'run' && /^(0|[1-9]\d*)$/.test(argument ?? '')) { await runShard(repositoryRoot, Number(argument)); diff --git a/scripts/full-mutations.test.mjs b/scripts/full-mutations.test.mjs index 6bf0e9c..961ed3f 100644 --- a/scripts/full-mutations.test.mjs +++ b/scripts/full-mutations.test.mjs @@ -5,7 +5,6 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { currentSources, - partitionSources, validateShardEvidence, assertShardTestInventory, } from './full-mutations.mjs'; @@ -13,19 +12,6 @@ import { sha256 } from './mutation-policy.mjs'; import { fingerprint } from './mutation-cache-inputs.mjs'; import { assertEncoderShardDiscovery, ENCODER_TEST_NAMES } from './verify-encoder-discovery.mjs'; -test('whole-file partitions are complete, deterministic and bounded without empty jobs', () => { - assert.throws(() => partitionSources([]), /Empty/); - assert.throws(() => partitionSources(['a', 'a']), /Duplicate/); - for (const count of [1, 2, 8, 11, 12, 25]) { - const files = Array.from({ length: count }, (_, index) => `src/mp3/${index}.ts`); - const scopes = partitionSources(files); - assert.equal(scopes.length, Math.min(11, count)); - assert.ok(scopes.every((scope) => scope.length > 0)); - assert.deepEqual(scopes.flat().sort(), files.toSorted()); - assert.deepEqual(partitionSources(files.toReversed()), scopes); - } -}); - test('native source discovery picks up new nested files and excludes declarations', async () => { const root = await mkdtemp(join(tmpdir(), 'mp3-shard-discovery-')); try { diff --git a/scripts/mutation-partitions.mjs b/scripts/mutation-partitions.mjs new file mode 100644 index 0000000..2ef593e --- /dev/null +++ b/scripts/mutation-partitions.mjs @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict'; +import { Instrumenter } from '@stryker-mutator/instrumenter'; +import nativePackage from '@stryker-mutator/instrumenter/package.json' with { type: 'json' }; + +const options = { excludedMutations: [], ignorers: [], plugins: null }; +const logger = { debug() {}, info() {}, warn() {}, trace() {}, isDebugEnabled: () => false }; +const position = (a, b) => a.line - b.line || a.column - b.column; +const lexical = (a, b) => (a < b ? -1 : Number(a > b)); +const identity = ({ fileName, location, mutatorName, replacement }) => + JSON.stringify([fileName, location.start, location.end, mutatorName, replacement]); +const identities = (mutants) => mutants.map(identity).sort(); + +async function instrument(files) { + const { mutants } = await new Instrumenter(logger).instrument(files, options); + assert.ok( + mutants.every((mutant) => mutant.status !== 'Ignored'), + 'Ignored native mutant', + ); + return mutants; +} + +function components(mutants) { + const result = []; + const sorted = mutants.toSorted( + (a, b) => + position(a.location.start, b.location.start) || position(a.location.end, b.location.end), + ); + for (const mutant of sorted) { + const previous = result.at(-1); + if (previous && position(mutant.location.start, previous.end) < 0) { + if (position(mutant.location.end, previous.end) > 0) previous.end = mutant.location.end; + previous.mutants.push(mutant); + } else result.push({ fileName: mutant.fileName, ...mutant.location, mutants: [mutant] }); + } + return result; +} + +function reportIdentity({ fileName, location, mutatorName, replacement }) { + const point = ({ line, column }) => ({ line: line + 1, column: column + 1 }); + return { + fileName, + location: { start: point(location.start), end: point(location.end) }, + mutatorName, + replacement, + }; +} + +async function provePartition(parts, sources) { + const ranges = new Map(); + for (const { fileName, start, end } of parts) { + if (!ranges.has(fileName)) ranges.set(fileName, []); + ranges.get(fileName).push({ start, end }); + } + const files = [...ranges].map(([name, mutate]) => ({ name, content: sources.get(name), mutate })); + const expected = parts.flatMap((part) => part.mutants); + assert.deepEqual( + identities(await instrument(files)), + identities(expected), + 'Native partition coverage differs', + ); + return expected.map(reportIdentity); +} + +// Native ranges include whole AST nodes. Overlapping mutant spans stay together; +// instrumenting the actual planned ranges proves that no boundary drops a mutant. +export async function mutationPartitions(sources, limit = 11) { + assert.equal(nativePackage.version, '10.1.1', 'Unsupported native instrumenter version'); + assert.ok(sources instanceof Map && sources.size > 0, 'Empty parser source inventory'); + assert.ok(Number.isInteger(limit) && limit > 0, 'Invalid parser shard limit'); + const parts = []; + const fullInventory = []; + for (const [name, content] of [...sources].sort(([a], [b]) => lexical(a, b))) { + assert.ok(typeof content === 'string' && content.length > 0, `Empty parser source: ${name}`); + const mutants = await instrument([{ name, content, mutate: true }]); + assert.ok(mutants.length > 0, `No native mutants: ${name}`); + assert.equal(new Set(identities(mutants)).size, mutants.length, 'Duplicate native mutant'); + fullInventory.push(...mutants.map(reportIdentity)); + parts.push(...components(mutants)); + } + parts.sort( + (a, b) => + b.mutants.length - a.mutants.length || + lexical(a.fileName, b.fileName) || + position(a.start, b.start), + ); + const length = Math.min(limit, parts.length); + const bins = Array.from({ length }, () => ({ parts: [], count: 0 })); + for (const part of parts) { + const bin = bins.reduce((a, b) => (a.count <= b.count ? a : b)); + bin.parts.push(part); + bin.count += part.mutants.length; + } + const scopes = bins.map((bin) => + bin.parts.map( + ({ fileName, start, end }) => + `${fileName}:${start.line + 1}:${start.column}-${end.line + 1}:${end.column}`, + ), + ); + const inventories = []; + for (const bin of bins) inventories.push(await provePartition(bin.parts, sources)); + assert.deepEqual( + identities(inventories.flat()), + identities(fullInventory), + 'Incomplete native partition union', + ); + return { scopes, inventories }; +} diff --git a/scripts/mutation-partitions.test.mjs b/scripts/mutation-partitions.test.mjs new file mode 100644 index 0000000..6218d6d --- /dev/null +++ b/scripts/mutation-partitions.test.mjs @@ -0,0 +1,156 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { Instrumenter } from '@stryker-mutator/instrumenter'; +import { mutationPartitions } from './mutation-partitions.mjs'; + +const fixture = `export class Probe { + first(n: number) { + if (n > 0) { return '🎵'; } + return 'no'; + } + second(n: number) { while (n > 0) n--; return n; } +} +export const positive = (n: number) => n > 0; +export const greeting = '🎵'; export const enabled = true; +`; +const logger = { debug() {}, info() {}, warn() {}, trace() {}, isDebugEnabled: () => false }; +const options = { excludedMutations: [], ignorers: [], plugins: null }; +const key = ({ fileName, location, mutatorName, replacement }) => + JSON.stringify([fileName, location, mutatorName, replacement]); +const keys = (mutants) => mutants.map(key).sort(); + +async function fullInventory(sources, scopes) { + const files = [...sources].flatMap(([name, content]) => { + if (!scopes) return [{ name, content, mutate: true }]; + const mutate = scopes.flatMap((scope) => { + const match = /^(.*):(\d+):(\d+)-(\d+):(\d+)$/.exec(scope); + assert.ok(match, `Invalid native CLI range: ${scope}`); + const [, file, startLine, startColumn, endLine, endColumn] = match; + if (file !== name) return []; + return [ + { + start: { line: Number(startLine) - 1, column: Number(startColumn) }, + end: { line: Number(endLine) - 1, column: Number(endColumn) }, + }, + ]; + }); + return mutate.length ? [{ name, content, mutate }] : []; + }); + const { mutants } = await new Instrumenter(logger).instrument(files, options); + return mutants.map(({ fileName, location, mutatorName, replacement }) => ({ + fileName, + location: Object.fromEntries( + Object.entries(location).map(([side, { line, column }]) => [ + side, + { line: line + 1, column: column + 1 }, + ]), + ), + mutatorName, + replacement, + })); +} + +test('native partitions preserve nested block, class method and Unicode mutant identities exactly once', async () => { + const sources = new Map([['src/probe.ts', fixture]]); + const { scopes, inventories } = await mutationPartitions(sources); + assert.ok(scopes.length > 1, 'Separate class methods should share work across runners'); + assert.ok(scopes.length <= 11); + assert.equal(scopes.length, inventories.length); + assert.ok(inventories.every((inventory) => inventory.length > 0)); + const observed = inventories.flat(); + assert.deepEqual(keys(observed), keys(await fullInventory(sources))); + assert.equal(new Set(keys(observed)).size, observed.length); + for (const [index, scope] of scopes.entries()) { + assert.deepEqual(keys(await fullInventory(sources, scope)), keys(inventories[index])); + } + // Removing a method body spans the nested condition: it must be in one range. + const block = observed.find( + (mutant) => mutant.location.start.line === 2 && mutant.replacement === '{}', + ); + assert.ok(block); + const owner = inventories.findIndex((inventory) => inventory.includes(block)); + assert.ok(inventories[owner].some((mutant) => mutant.location.start.line === 3)); + assert.ok(observed.some((mutant) => mutant.location.start.column > 30)); +}); + +test('assignment is deterministic across source insertion orders and respects runner limits', async () => { + const entries = [ + ['src/z.ts', fixture], + ['src/A.ts', fixture], + ['src/a.ts', fixture], + ]; + const sources = new Map(entries); + const forward = await mutationPartitions(sources, 3); + const reverse = await mutationPartitions(new Map(entries.toReversed()), 3); + assert.deepEqual(reverse, forward); + assert.equal(forward.scopes.length, 3); + assert.deepEqual(keys(forward.inventories.flat()), keys(await fullInventory(sources))); + const single = await mutationPartitions(sources, 1); + assert.equal(single.scopes.length, 1); + assert.deepEqual(keys(single.inventories.flat()), keys(forward.inventories.flat())); +}); + +test('one indivisible function stays whole even when many runners are requested', async () => { + const sources = new Map([ + ['src/one.ts', 'export function f(n) { if (n > 0) return n + 1; return 0; }'], + ]); + const result = await mutationPartitions(sources, 11); + assert.equal(result.scopes.length, 1); + assert.deepEqual(keys(result.inventories.flat()), keys(await fullInventory(sources))); +}); + +test('empty sources, missing mutants, ignored mutants and invalid limits fail closed', async () => { + await assert.rejects(mutationPartitions(new Map()), /Empty parser source inventory/); + await assert.rejects(mutationPartitions(new Map([['src/empty.ts', '']])), /Empty parser source/); + await assert.rejects( + mutationPartitions(new Map([['src/types.ts', 'export type Number = number;']])), + /No native mutants/, + ); + await assert.rejects( + mutationPartitions( + new Map([['src/ignored.ts', '// Stryker disable all\nexport const flag = true;']]), + ), + /Ignored native mutant/, + ); + for (const limit of [0, -1, 1.5, NaN]) { + await assert.rejects( + mutationPartitions(new Map([['src/probe.ts', fixture]]), limit), + /Invalid parser shard limit/, + ); + } +}); + +test('native re-instrumentation rejects missing, duplicate and extra partition coverage', async (t) => { + const corruptions = { + missing: (mutants) => mutants.slice(1), + duplicate: (mutants) => [...mutants, mutants[0]], + extra: (mutants) => [{ ...mutants[0], replacement: 'unexpected' }, ...mutants.slice(1)], + }; + for (const [name, corrupt] of Object.entries(corruptions)) { + await t.test(name, async (t) => { + const original = Instrumenter.prototype.instrument; + t.mock.method(Instrumenter.prototype, 'instrument', async function (files, options) { + const result = await original.call(this, files, options); + return Array.isArray(files[0].mutate) + ? { ...result, mutants: corrupt(result.mutants) } + : result; + }); + await assert.rejects( + mutationPartitions(new Map([['src/probe.ts', fixture]])), + /Native partition coverage differs/, + ); + }); + } +}); + +test('ambiguous duplicate identities in native full inventory fail closed', async (t) => { + const original = Instrumenter.prototype.instrument; + t.mock.method(Instrumenter.prototype, 'instrument', async function (files, options) { + const result = await original.call(this, files, options); + return { ...result, mutants: [...result.mutants, result.mutants[0]] }; + }); + await assert.rejects( + mutationPartitions(new Map([['src/probe.ts', fixture]])), + /Duplicate native mutant/, + ); +}); diff --git a/scripts/mutation-policy-inputs.mjs b/scripts/mutation-policy-inputs.mjs index a36eee7..785475c 100644 --- a/scripts/mutation-policy-inputs.mjs +++ b/scripts/mutation-policy-inputs.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { isDeepStrictEqual } from 'node:util'; export const PARSER_GLOB = 'src/mp3/**/*.ts'; export const VITEST_CONFIG = 'vitest.mutation.config.ts'; @@ -60,3 +61,32 @@ export function validateSourceInventory(inventory, policy, sources) { ); return paths; } + +export function validatePartitionInventory(reports, inventories) { + requireThat(inventories.length === reports.length, 'Missing native partition inventory'); + const key = (file, { location: { start, end }, mutatorName, replacement }) => + JSON.stringify([ + file, + start.line, + start.column, + end.line, + end.column, + mutatorName, + replacement, + ]); + const seen = new Set(); + for (const [index, expected] of inventories.entries()) { + const expectedKeys = expected.map(({ fileName, ...mutant }) => key(fileName, mutant)); + for (const identity of expectedKeys) { + requireThat(!seen.has(identity), 'Duplicate native partition ownership'); + seen.add(identity); + } + const actual = Object.entries(reports[index].files).flatMap(([file, record]) => + record.mutants.map((mutant) => key(file, mutant)), + ); + requireThat( + isDeepStrictEqual(actual.sort(), expectedKeys.sort()), + 'Missing, extra or duplicate native partition mutant', + ); + } +} diff --git a/scripts/mutation-policy.mjs b/scripts/mutation-policy.mjs index 38b7e22..54c5160 100644 --- a/scripts/mutation-policy.mjs +++ b/scripts/mutation-policy.mjs @@ -5,6 +5,7 @@ import { validateSchemas, validateMutationConfiguration, validateSourceInventory, + validatePartitionInventory, } from './mutation-policy-inputs.mjs'; export { PARSER_GLOB, VITEST_CONFIG, sha256, sourceContext } from './mutation-policy-inputs.mjs'; @@ -232,7 +233,10 @@ function validateShard(report, scope, referenceConfig, policy) { 'Inconsistent shard execution configurations', ); requireThat( - isDeepStrictEqual(Object.keys(report.files ?? {}).sort(), scope.toSorted()), + isDeepStrictEqual( + Object.keys(report.files ?? {}).sort(), + [...new Set(scope.map((range) => range.split(':')[0]))].sort(), + ), 'Shard report does not match its assigned source files', ); for (const record of Object.values(report.files)) @@ -244,7 +248,7 @@ function validateShard(report, scope, referenceConfig, policy) { } /** Audit disjoint raw reports without merging their local mutant or test IDs. */ -export function auditMutationShards(reports, policy, sources, scopes) { +export function auditMutationShards(reports, policy, sources, scopes, inventories) { requireThat( Array.isArray(scopes) && scopes.length > 0 && @@ -255,10 +259,14 @@ export function auditMutationShards(reports, policy, sources, scopes) { Array.isArray(reports) && reports.length === scopes.length, 'Missing or extra shard reports', ); - const paths = validateSourceInventory(scopes.flat(), policy, sources); + const files = inventories + ? [...new Set(scopes.flat().map((range) => range.split(':')[0]))] + : scopes.flat(); + const paths = validateSourceInventory(files, policy, sources); reports.forEach((report, index) => validateShard(report, scopes[index], reports[0]?.config, policy), ); + if (inventories) validatePartitionInventory(reports, inventories); return auditReports(reports, policy, sources, paths, false); } @@ -277,7 +285,8 @@ function auditReports(reports, policy, sources, paths, nativeReuse) { context.mutantIds = new Set(); for (const file of Object.keys(report.files).sort()) { const counts = auditSourceMutants(file, sources.get(file), report.files[file], context); - perFile[file] = counts; + perFile[file] ??= { killed: 0, timedOut: 0, equivalent: 0, total: 0 }; + for (const key of Object.keys(counts)) perFile[file][key] += counts[key]; for (const k of Object.keys(totals)) totals[k] += counts[k]; } } diff --git a/scripts/mutation-shards-policy.test.mjs b/scripts/mutation-shards-policy.test.mjs index e0069ce..ffde4ca 100644 --- a/scripts/mutation-shards-policy.test.mjs +++ b/scripts/mutation-shards-policy.test.mjs @@ -74,8 +74,8 @@ function fixture(count = 2) { return { sources, policy, scopes, reports }; } -const run = ({ reports, policy, sources, scopes }) => - auditMutationShards(reports, policy, sources, scopes); +const run = ({ reports, policy, sources, scopes, inventories }) => + auditMutationShards(reports, policy, sources, scopes, inventories); const record = (f, index = 0) => f.reports[index].files[f.scopes[index][0]]; const rejected = (change, message) => { const f = fixture(); @@ -335,6 +335,70 @@ test('reviewed timeout accounting stays global and excludes it from adjusted det assert.deepEqual(f, before); }); +function rangeFixture() { + const f = fixture(1); + const file = [...f.sources.keys()][0]; + const original = f.reports[0]; + f.reports = original.files[file].mutants.map((mutant) => { + const report = structuredClone(original); + report.files[file].mutants = [structuredClone(mutant)]; + const { start, end } = mutant.location; + report.config.mutate = [ + `${file}:${start.line}:${start.column - 1}-${end.line}:${end.column - 1}`, + ]; + return report; + }); + f.scopes = f.reports.map((report) => report.config.mutate); + f.inventories = f.reports.map((report) => + report.files[file].mutants.map(({ location, mutatorName, replacement }) => ({ + fileName: file, + location, + mutatorName, + replacement, + })), + ); + return f; +} + +test('range reports preserve source-bound proofs and aggregate per-file counts without rewriting', () => { + const f = rangeFixture(); + const before = structuredClone(f); + const result = run(f); + assert.equal(result.total, 2); + assert.deepEqual(result.perFile['src/mp3/file-0.ts'], { + killed: 1, + timedOut: 0, + equivalent: 1, + total: 2, + }); + assert.equal(result.adjustedScore, 100); + assert.deepEqual(f, before); +}); + +test('range collection rejects missing, duplicate and misplaced native identities', () => { + for (const change of [ + (f, file) => { + f.reports[0].files[file].mutants.push(f.reports[1].files[file].mutants[0]); + }, + (f, file) => { + f.reports[0].files[file].mutants = f.reports[1].files[file].mutants; + }, + (f, file) => { + f.reports[0].files[file].mutants[0].replacement = 'unplanned'; + }, + (f) => { + f.inventories[1] = f.inventories[0]; + }, + (f) => { + f.inventories.pop(); + }, + ]) { + const f = rangeFixture(); + change(f, [...f.sources.keys()][0]); + assert.throws(() => run(f), /native partition/); + } +}); + for (const status of [ 'NoCoverage', 'Ignored', diff --git a/scripts/mutation-workflow.test.mjs b/scripts/mutation-workflow.test.mjs index dd01b0c..c51f4c4 100644 --- a/scripts/mutation-workflow.test.mjs +++ b/scripts/mutation-workflow.test.mjs @@ -71,10 +71,8 @@ test('full partitions use a native dynamic matrix capped at eleven beside the pu }); assert.deepEqual(plan.outputs, { matrix: '${{ steps.plan.outputs.matrix }}' }); assert.deepEqual( - plan.steps.find((step) => step.uses === './.github/actions/setup-project').with, - { - install: 'false', - }, + plan.steps.find((step) => step.uses === './.github/actions/setup-project'), + { uses: './.github/actions/setup-project' }, ); const planning = plan.steps.find((step) => step.id === 'plan').run; assert.ok(planning.startsWith('matrix=$(node scripts/full-mutations.mjs plan)\n'));