diff --git a/.github/workflows/mutation-affected.yml b/.github/workflows/mutation-affected.yml index 2307c2f2cc..6886d8f5bc 100644 --- a/.github/workflows/mutation-affected.yml +++ b/.github/workflows/mutation-affected.yml @@ -1,4 +1,4 @@ -name: Mutation Affected +name: Mutation Harness Canary # PR-side half of the decision-kernel mutation lane (issue #1415). # @@ -9,7 +9,7 @@ name: Mutation Affected # spends mutants on one thing only — proving the harness still runs end to end # when the harness itself changes. `select` therefore returns an empty matrix # unless the diff touches the lane's own sources. Selecting on derived kernel -# ownership instead would run the full ten-shard sweep on 24 of the last 40 +# ownership instead would run the full sweep on 24 of the last 40 # merged PRs. # # On a harness diff the matrix is the canary (`LANE_CANARY`, selected by @@ -22,14 +22,20 @@ name: Mutation Affected on: pull_request: paths: - # Exactly `LANE_TOOLING` in scripts/mutation/run.ts plus this file — the + # Exactly `LANE_TOOLING` in scripts/mutation/run.ts — the # only diffs that can produce a non-empty matrix, asserted both ways by # scripts/mutation/workflow.test.ts. A kernel or test path here would only # start a job that selects nothing. - 'scripts/mutation/**' - 'scripts/lib/**' - 'stryker.config.json' + - 'vitest.mutation.config.ts' + - 'vitest.config.ts' + - '.github/workflows/mutation-weekly.yml' - '.github/workflows/mutation-affected.yml' + - 'package.json' + - 'pnpm-lock.yaml' + - 'packages/*/package.json' permissions: contents: read @@ -40,7 +46,7 @@ concurrency: jobs: select: - name: Select affected kernels + name: Select harness validation kernels runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -71,13 +77,13 @@ jobs: # The self-test and the derivation run before any mutant, so a failure here # would leave the lane with no envelope at all (#1430). - name: Record a failed lane envelope - if: failure() + if: failure() || cancelled() run: | pnpm gate mutation --affected --fail-envelope \ "affected selection failed before any mutant ran (run ${{ github.run_id }})" || true - name: Upload selection envelope - if: failure() + if: failure() || cancelled() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: mutation-affected-select @@ -97,12 +103,15 @@ jobs: include: ${{ fromJSON(needs.select.outputs.modules) }} steps: - name: Checkout + timeout-minutes: 1 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain + timeout-minutes: 2 uses: ./.github/actions/setup-node-pnpm - name: Run mutants for ${{ matrix.name }} + timeout-minutes: 22 uses: ./.github/actions/run-gate with: gate: mutation @@ -113,7 +122,7 @@ jobs: ${{ matrix.shard || '' }} - name: Record a failed shard envelope - if: failure() + if: failure() || cancelled() run: | pnpm gate mutation --affected --modules ${{ matrix.module }} --fail-envelope \ "shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true @@ -167,7 +176,7 @@ jobs: fi - name: Record a failed lane envelope - if: failure() + if: failure() || cancelled() run: | pnpm gate mutation --affected --fail-envelope \ "affected report failed before producing a score table (run ${{ github.run_id }})" || true diff --git a/.github/workflows/mutation-weekly.yml b/.github/workflows/mutation-weekly.yml index ed8ddf701b..42c93e1318 100644 --- a/.github/workflows/mutation-weekly.yml +++ b/.github/workflows/mutation-weekly.yml @@ -5,10 +5,8 @@ name: Mutation Weekly # decorative"; a full-suite sweep is unaffordable, so the scope is the kernel # registry in scripts/mutation/modules.ts and nothing else. # -# Sharded one job per module: the whole sweep is ~2,150 mutants, and the selector -# module alone is ~1,280 of them, so a single job would sit near the 30-minute -# acceptance budget on an ubuntu runner. The shards' JSON reports are merged into -# one score table by the report job (`--report-dir`). +# Sharded from the decision-kernel registry. Oversized sources are partitioned +# at syntax boundaries; the report job merges all required shards into one table. # # The lane reports and never gates (#1457): the per-kernel table lands in the job # summary and the artifact, and a low score is an input for a human-authored @@ -31,9 +29,7 @@ jobs: shard: name: Mutants (${{ matrix.name }}) runs-on: ubuntu-latest - # The acceptance budget is 30 minutes of wall clock for the lane, and shards - # run in parallel, so the budget is per shard. The observed rate on a 2-core - # runner is ~3s/mutant, which is why selectors (~1,280 mutants) is sliced. + # Bound setup and mutation separately, leaving headroom for evidence and upload. timeout-minutes: 30 strategy: fail-fast: false @@ -42,24 +38,35 @@ jobs: # scripts/mutation/workflow.test.ts, which also pins --expect-shards to # the number of entries here. include: - - { name: kernel-errors, module: kernel-errors } + - { name: kernel-errors-1, module: kernel-errors, shard: 1/5 } + - { name: kernel-errors-2, module: kernel-errors, shard: 2/5 } + - { name: kernel-errors-3, module: kernel-errors, shard: 3/5 } + - { name: kernel-errors-4, module: kernel-errors, shard: 4/5 } + - { name: kernel-errors-5, module: kernel-errors, shard: 5/5 } - { name: daemon-ref-frame, module: daemon-ref-frame } - { name: interaction-settle, module: interaction-settle } - { name: scroll-edge-state, module: scroll-edge-state } - - { name: selectors-1, module: selectors, shard: 1/4 } - - { name: selectors-2, module: selectors, shard: 2/4 } - - { name: selectors-3, module: selectors, shard: 3/4 } - - { name: selectors-4, module: selectors, shard: 4/4 } + - { name: selectors-1, module: selectors, shard: 1/8 } + - { name: selectors-2, module: selectors, shard: 2/8 } + - { name: selectors-3, module: selectors, shard: 3/8 } + - { name: selectors-4, module: selectors, shard: 4/8 } + - { name: selectors-5, module: selectors, shard: 5/8 } + - { name: selectors-6, module: selectors, shard: 6/8 } + - { name: selectors-7, module: selectors, shard: 7/8 } + - { name: selectors-8, module: selectors, shard: 8/8 } - { name: target-annotation-serde, module: target-annotation-serde } - { name: snapshot-occlusion, module: snapshot-occlusion } steps: - name: Checkout + timeout-minutes: 1 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain + timeout-minutes: 2 uses: ./.github/actions/setup-node-pnpm - name: Run mutants for ${{ matrix.name }} + timeout-minutes: 22 uses: ./.github/actions/run-gate with: gate: mutation @@ -73,7 +80,7 @@ jobs: # without this the artifact is absent and "shard failed" is indistinguishable # from "lane never ran". --fail-envelope leaves a real verdict untouched. - name: Record a failed shard envelope - if: failure() + if: failure() || cancelled() run: | pnpm gate mutation --modules ${{ matrix.module }} --fail-envelope \ "shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true @@ -126,12 +133,12 @@ jobs: --report-dir .tmp/mutation/shards --expect-shards - 10 + 18 # The self-test and the artifact download both run before the scoring step, # so a failure there would otherwise leave the aggregate lane with no envelope. - name: Record a failed lane envelope - if: failure() + if: failure() || cancelled() run: | pnpm gate mutation --fail-envelope \ "weekly report job failed before producing a score table (run ${{ github.run_id }})" || true diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 01c4e2a21b..ce43016caa 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -125,8 +125,8 @@ worker process, so a case that never returns or kills its process is reported as against the exact input. Promote a discovered case with the command the harness prints — never hand-copy an unshrunk input. -Mutation is report-only and limited to the registry in `scripts/mutation/modules.ts`. It measures -whether tests distinguish changed decision logic. Do not infer redundancy from line coverage alone. +Scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` (PR canary). +Scores are advisory; incomplete sweeps fail. Coverage does not prove test strength. ## Before editing a shared module diff --git a/package.json b/package.json index 76cbf59d65..f95198cb54 100644 --- a/package.json +++ b/package.json @@ -321,6 +321,7 @@ "@nkzw/oxlint-config": "^1.3.0", "@statelyai/graph": "2.4.0", "@stryker-mutator/core": "10.0.0", + "@stryker-mutator/instrumenter": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", "@types/json-schema": "^7.0.15", "@types/node": "^22.20.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 824dcad782..a4dbfc4ed3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -122,6 +122,9 @@ importers: '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) + '@stryker-mutator/instrumenter': + specifier: 10.0.0 + version: 10.0.0 '@stryker-mutator/vitest-runner': specifier: 10.0.0 version: 10.0.0(@stryker-mutator/core@10.0.0(@types/node@22.20.1))(vitest@4.1.11) diff --git a/scripts/mutation/config.test.ts b/scripts/mutation/config.test.ts index c6896a5735..c981f2014a 100644 --- a/scripts/mutation/config.test.ts +++ b/scripts/mutation/config.test.ts @@ -39,3 +39,20 @@ test('the config content hash is stable and content-addressed', () => { assert.notEqual(configHash('a'), configHash('b')); assert.match(configHash('a'), /^sha256:[0-9a-f]{12}$/); }); + +test('stryker disables per-mutant related discovery and uses perTest coverage', () => { + const config = readConfig(); + assert.equal((config.vitest as { related: boolean }).related, false); + assert.equal(config.coverageAnalysis, 'perTest'); +}); + +test('shard planning pins the same instrumenter version as Stryker core', () => { + const root = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')); + const core = JSON.parse( + fs.readFileSync(path.join(repoRoot, 'node_modules/@stryker-mutator/core/package.json'), 'utf8'), + ); + assert.equal( + root.devDependencies['@stryker-mutator/instrumenter'], + core.dependencies['@stryker-mutator/instrumenter'], + ); +}); diff --git a/scripts/mutation/envelope.test.ts b/scripts/mutation/envelope.test.ts index f088c5146e..a0f2b67607 100644 --- a/scripts/mutation/envelope.test.ts +++ b/scripts/mutation/envelope.test.ts @@ -6,6 +6,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; import { test } from 'node:test'; import { runCmdSync } from '@agent-device/host-kit/command'; import { laneEnvelope, LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope.ts'; @@ -86,6 +87,9 @@ test('a crashed run still writes an envelope naming the stage it died in', () => assert.equal(envelope.data.stage, 'report'); assert.match(envelope.data.error ?? '', /absent-report\.json/); // Provenance is read before the work, so a crash still reports its tool/config. + const unmeasured = JSON.parse(fs.readFileSync(envelopePath, 'utf8')).data.modules[0]; + assert.equal(unmeasured.noCoverage, null); + assert.equal(unmeasured.errors, null); assert.ok(envelope.tool.stryker); assert.match(envelope.configHash, /^sha256:/); }); @@ -182,6 +186,7 @@ test('an incomplete shard set fails instead of scoring the missing module as zer files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } }, }), ); + fs.writeFileSync(path.join(shards, 'lane-envelope.json'), JSON.stringify({ result: 'pass' })); const result = runMutation([ '--report-dir', '.tmp/mutation/partial-shards', @@ -222,6 +227,8 @@ test('a short shard set publishes the shards that did report, then fails', () => ]); assert.notEqual(result.exitCode, 0, 'a short shard set must fail the aggregate'); assert.match(result.stderr, /1 report\(s\), expected 10/); + assert.match(result.stdout, /Incomplete sweep: Incomplete shard set.*1 report\(s\), expected 10/); + assert.doesNotMatch(result.stdout, /this lane never fails a build/); assert.match(result.stdout, /\| `kernel-errors` — [^|]+\| 100% \|/); const envelope = readEnvelope(); assert.equal(envelope.result, 'fail'); @@ -276,6 +283,7 @@ test('--fail-envelope downgrades a passing envelope when a later step fails', () files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Survived' }] } }, }), ); + fs.writeFileSync(path.join(shards, 'lane-envelope.json'), JSON.stringify({ result: 'pass' })); const passing = runMutation([ '--report-dir', '.tmp/mutation/pass-then-fail', @@ -308,3 +316,124 @@ test('both mutation lanes publish the envelope', () => { } assert.match(workflow('mutation-weekly.yml'), /Lane envelope/); }); + +test('an envelope without a report cannot disappear from an otherwise healthy aggregate', () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-envelope-only-')); + try { + const healthy = path.join(root, 'healthy'); + const failed = path.join(root, 'failed-without-report'); + fs.mkdirSync(healthy); + fs.mkdirSync(failed); + fs.writeFileSync( + path.join(healthy, 'mutation.json'), + JSON.stringify({ + files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } }, + }), + ); + fs.writeFileSync(path.join(healthy, 'lane-envelope.json'), JSON.stringify({ result: 'pass' })); + for (const verdict of ['fail', 'pass']) { + fs.writeFileSync( + path.join(failed, 'lane-envelope.json'), + JSON.stringify({ result: verdict }), + ); + const result = runMutation(['--report-dir', root, '--modules', 'kernel-errors']); + assert.notEqual(result.exitCode, 0); + assert.match( + result.stdout, + /Incomplete sweep: Shard failure or missing envelope: failed-without-report/, + ); + assert.equal(readEnvelope().result, 'fail'); + } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('a report from a failed shard cannot make the aggregate pass', () => { + const dir = '.tmp/mutation/failed-shard'; + const shard = path.join(repoRoot, dir, 'shard-a'); + fs.mkdirSync(shard, { recursive: true }); + fs.writeFileSync( + path.join(shard, 'mutation.json'), + JSON.stringify({ + files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } }, + }), + ); + try { + for (const countArgs of [[], ['--expect-shards', '1']]) { + const args = ['--report-dir', dir, ...countArgs, '--modules', 'kernel-errors']; + fs.writeFileSync(path.join(shard, 'lane-envelope.json'), JSON.stringify({ result: 'fail' })); + const failed = runMutation(args); + assert.notEqual(failed.exitCode, 0); + const reason = 'Shard failure or missing envelope: shard-a'; + assert.match(failed.stdout, new RegExp(`Incomplete sweep: ${reason}`)); + assert.deepEqual( + failed.stderr.split('\n').filter((line) => line.startsWith('mutation: ')), + [`mutation: ${reason}`], + ); + assert.equal(readEnvelope().data.error, reason); + fs.writeFileSync(path.join(shard, 'lane-envelope.json'), JSON.stringify({ result: 'pass' })); + const passed = runMutation(args); + assert.equal(passed.exitCode, 0, passed.stderr); + assert.doesNotMatch(passed.stdout, /Incomplete sweep/); + fs.rmSync(path.join(shard, 'lane-envelope.json')); + const missing = runMutation(args); + assert.notEqual(missing.exitCode, 0); + } + } finally { + fs.rmSync(path.join(repoRoot, dir), { recursive: true, force: true }); + } +}); + +test('a non-zero Stryker exit fails even when it wrote a report', () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-failed-exit-')); + try { + runCmdSync('git', ['init', '--quiet', root]); + fs.mkdirSync(path.join(root, 'src')); + fs.mkdirSync(path.join(root, 'packages/kernel/src'), { recursive: true }); + fs.mkdirSync(path.join(root, 'bin')); + fs.symlinkSync(path.join(repoRoot, 'node_modules'), path.join(root, 'node_modules')); + fs.writeFileSync(path.join(root, 'stryker.config.json'), '{}'); + fs.writeFileSync( + path.join(root, 'vitest.config.ts'), + `export default { test: { projects: [{ test: { name: 'unit-core', include: ['src/**/*.test.ts'] } }] } };`, + ); + fs.writeFileSync(path.join(root, 'packages/kernel/src/errors.ts'), 'export const value = 1;'); + fs.writeFileSync( + path.join(root, 'src/consumer.test.ts'), + "import { value } from '../packages/kernel/src/errors'; console.log(value);", + ); + fs.writeFileSync( + path.join(root, 'bin/pnpm'), + `#!/bin/sh +mkdir -p .tmp/mutation +printf '{"files":{}}' > .tmp/mutation/mutation.json +exit 1 +`, + { mode: 0o755 }, + ); + const result = runCmdSync( + process.execPath, + [ + '--experimental-strip-types', + path.join(repoRoot, 'scripts/mutation/run.ts'), + '--modules', + 'kernel-errors', + ], + { + cwd: root, + env: { ...process.env, PATH: `${path.join(root, 'bin')}:${process.env.PATH}` }, + allowFailure: true, + }, + ); + assert.notEqual(result.exitCode, 0); + assert.match(result.stderr, /Stryker failed \(exit 1\)/); + const envelope = JSON.parse( + fs.readFileSync(path.join(root, '.tmp/mutation/lane-envelope.json'), 'utf8'), + ); + assert.equal(envelope.result, 'fail'); + assert.equal(envelope.data.stage, 'stryker'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/mutation/modules.test.ts b/scripts/mutation/modules.test.ts index 5f4f0b7935..370296bd19 100644 --- a/scripts/mutation/modules.test.ts +++ b/scripts/mutation/modules.test.ts @@ -59,15 +59,21 @@ test('mutate globs default to every module and narrow on request', () => { // the shard count is registry data so the workflows and the runner agree on it. test('the shard matrix slices only the modules that declare shards', () => { assert.deepEqual(shardMatrix(['kernel-errors']), [ - { name: 'kernel-errors', module: 'kernel-errors' }, + { name: 'kernel-errors-1', module: 'kernel-errors', shard: '1/5' }, + { name: 'kernel-errors-2', module: 'kernel-errors', shard: '2/5' }, + { name: 'kernel-errors-3', module: 'kernel-errors', shard: '3/5' }, + { name: 'kernel-errors-4', module: 'kernel-errors', shard: '4/5' }, + { name: 'kernel-errors-5', module: 'kernel-errors', shard: '5/5' }, ]); - assert.deepEqual(shardMatrix(['selectors']), [ - { name: 'selectors-1', module: 'selectors', shard: '1/4' }, - { name: 'selectors-2', module: 'selectors', shard: '2/4' }, - { name: 'selectors-3', module: 'selectors', shard: '3/4' }, - { name: 'selectors-4', module: 'selectors', shard: '4/4' }, - ]); - assert.equal(shardMatrix().length, ALL_MODULE_IDS.length + 3); + assert.deepEqual( + shardMatrix(['selectors']), + Array.from({ length: 8 }, (_, index) => ({ + name: `selectors-${index + 1}`, + module: 'selectors', + shard: `${index + 1}/8`, + })), + ); + assert.equal(shardMatrix().length, ALL_MODULE_IDS.length + 11); // Every registry module reaches the matrix: an unsharded sweep is not a sweep. for (const id of ALL_MODULE_IDS) { assert.ok( diff --git a/scripts/mutation/modules.ts b/scripts/mutation/modules.ts index d0a6f24939..62962325ef 100644 --- a/scripts/mutation/modules.ts +++ b/scripts/mutation/modules.ts @@ -45,6 +45,7 @@ export const KERNEL_MODULES: readonly KernelModule[] = [ label: 'Error retriability + hints', mutate: ['packages/kernel/src/errors.ts'], owns: ['packages/kernel/src/errors.ts'], + shards: 5, }, { id: 'daemon-ref-frame', @@ -76,13 +77,12 @@ export const KERNEL_MODULES: readonly KernelModule[] = [ mutate: [ 'packages/selectors/src/**/*.ts', '!packages/selectors/src/**/*.test.ts', + '!packages/selectors/src/**/*.fixtures.ts', '!packages/selectors/src/internal/__tests__/**', ], // Selector tests live under the owned package source, so the prefix covers them. owns: ['packages/selectors/src/'], - // ~1,280 mutants at the observed ~3s/mutant on a 2-core runner is ~64 - // minutes in one job — past the acceptance budget and its own timeout. - shards: 4, + shards: 8, }, { id: 'target-annotation-serde', diff --git a/scripts/mutation/report.test.ts b/scripts/mutation/report.test.ts index 74202f95cb..4cbbfd919f 100644 --- a/scripts/mutation/report.test.ts +++ b/scripts/mutation/report.test.ts @@ -37,6 +37,7 @@ test('the table carries score, killed, survived, total and timeouts per kernel', }, ['kernel-errors'], ); + assert.equal(scores[0]?.noCoverage, 1); const markdown = renderReport(scores, PROVENANCE); // Timeouts count as killed, so a score propped up by slow mutants rather than // assertions is only visible if the column is rendered. @@ -66,6 +67,7 @@ test('statuses outside the score (Ignored, CompileError, RuntimeError) leave the ['kernel-errors'], ); assert.equal(score?.total, 1); + assert.equal(score?.errors, 2); assert.equal(score?.score, 100); }); diff --git a/scripts/mutation/report.ts b/scripts/mutation/report.ts index 7da43dc0ca..8e838fbcd8 100644 --- a/scripts/mutation/report.ts +++ b/scripts/mutation/report.ts @@ -16,7 +16,7 @@ function renderRow(score: ModuleScore): string { const module = moduleById(score.module); return ( `| \`${score.module}\` — ${module.label} | ${score.score}% | ${score.killed} | ` + - `${score.survived} | ${score.total} | ${score.timeout} |` + `${score.survived} | ${score.total} | ${score.timeout} | ${score.noCoverage} | ${score.errors} |` ); } @@ -27,7 +27,7 @@ function renderDetail(score: ModuleScore, maxListed: number): string[] { '', ]; for (const mutant of score.surviving.slice(0, maxListed)) { - lines.push(`- \`${mutant.file}:${mutant.line}\` ${mutant.mutator}`); + lines.push(`- \`${mutant.file}:${mutant.line}\` ${mutant.mutator} (${mutant.status})`); } if (score.surviving.length > maxListed) { lines.push(`- …and ${score.surviving.length - maxListed} more`); @@ -37,6 +37,7 @@ function renderDetail(score: ModuleScore, maxListed: number): string[] { export type RenderOptions = { readonly title?: string; + readonly incomplete?: string; readonly maxSurvivingListed?: number; }; @@ -50,10 +51,11 @@ export function renderReport( `## ${options.title ?? 'Mutation score — decision kernels'}`, '', `Stryker \`${provenance.strykerVersion}\` · config \`${provenance.configHash}\` · ` + - 'report only — this lane never fails a build.', + 'scores are advisory; harness failures fail the lane.', '', - '| Kernel | Score | Killed | Survived | Total | Timeout |', - '| --- | --- | --- | --- | --- | --- |', + ...(options.incomplete ? [`**Incomplete sweep: ${options.incomplete}**`, ''] : []), + '| Kernel | Score | Killed | Survived | Total | Timeout | Uncovered | Errors |', + '| --- | --- | --- | --- | --- | --- | --- | --- |', ...scores.map(renderRow), ]; diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index a54bf59f03..76a759bd4f 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -28,6 +28,7 @@ import { type ModuleId, type ShardSpec, } from './modules.ts'; +import { shardFiles, type Shard } from './sharding.ts'; import { derivedAffectedModules } from './ownership.ts'; import { renderReport, type Provenance } from './report.ts'; import { mergeReports, summarizeReport, type ModuleScore, type StrykerReport } from './score.ts'; @@ -79,9 +80,6 @@ type Args = { expectShards: number | undefined; }; -/** One-based slice of a module's mutated sources: `--shard 2/4`. */ -type Shard = { index: number; count: number }; - function parseShard(value: string | undefined): Shard | undefined { if (!value) return undefined; const match = /^(\d+)\/(\d+)$/.exec(value.trim()); @@ -160,30 +158,6 @@ function changedFiles(base: string): string[] { return result.stdout.split('\n').filter(Boolean); } -// The JSON report path comes from the config (Stryker's CLI takes no nested -// reporter options), so a run always writes DEFAULT_REPORT_PATH. -/** - * Balances a module's sources across `count` jobs. Mutant count tracks file size - * closely enough that greedy longest-first packing keeps the slowest slice near - * the mean — the alternative is one 1,277-mutant selectors job that outruns both - * the 30-minute acceptance budget and its own timeout. - */ -function shardFiles(files: readonly string[], shard: Shard, root: string): string[] { - const bins: { size: number; files: string[] }[] = Array.from({ length: shard.count }, () => ({ - size: 0, - files: [], - })); - const weighed = files - .map((file) => ({ file, size: fs.statSync(path.join(root, file)).size })) - .sort((a, b) => b.size - a.size || a.file.localeCompare(b.file)); - for (const { file, size } of weighed) { - const bin = bins.reduce((smallest, next) => (next.size < smallest.size ? next : smallest)); - bin.files.push(file); - bin.size += size; - } - return bins[shard.index - 1]!.files.sort(); -} - async function runStryker( modules: readonly ModuleId[], reportPath: string, @@ -196,11 +170,18 @@ async function runStryker( // over the mutated files, not hand-listed; see scripts/mutation/test-scope.ts. const globs = mutateGlobs(modules); const all = expandMutateFiles(globs, repoRoot); - // A sharded job mutates concrete files, so the slice is exact rather than a - // glob the next contributor has to keep in step with the registry. - const mutate = shard ? shardFiles(all, shard, repoRoot) : globs; + // Source ranges keep single-file kernels inside the same shard budget. + const excludedMutations = + JSON.parse(fs.readFileSync(path.join(repoRoot, CONFIG_PATH), 'utf8')).mutator + ?.excludedMutations ?? []; + const mutate = shard ? await shardFiles(all, shard, repoRoot, excludedMutations) : globs; const scopePath = path.join(repoRoot, TEST_SCOPE_PATH); - const testFiles = relatedTestFiles(shard ? mutate : all, repoRoot); + const testFiles = await relatedTestFiles( + shard ? [...new Set(mutate.map((file) => file.split(':')[0]!))] : all, + repoRoot, + ); + if (testFiles.length === 0) + throw new Error(`No related mutation tests for ${modules.join(', ')}`); writeTestScope(testFiles, scopePath); process.stdout.write( `mutation: ${modules.join(', ')}${shard ? ` shard ${shard.index}/${shard.count}` : ''} -> ` + @@ -216,11 +197,11 @@ async function runStryker( onStdoutChunk: (chunk) => void process.stdout.write(chunk), onStderrChunk: (chunk) => void process.stderr.write(chunk), }); - // Stryker exits non-zero on a low score too; this lane reports scores rather - // than judging them, so only a missing report is fatal here. - if (!fs.existsSync(absolute)) { + // No score threshold is configured. Any non-zero exit is a harness failure, + // even if Stryker wrote a partial report before failing. + if (result.exitCode !== 0 || !fs.existsSync(absolute)) { throw new Error( - `Stryker produced no report at ${reportPath} (exit ${result.exitCode}). See output above.`, + `Stryker failed (exit ${result.exitCode}) or produced no report at ${reportPath}. See output above.`, ); } } @@ -282,7 +263,16 @@ function existingResult(file: string): string | undefined { function envelopeModule(id: ModuleId, scores: readonly ModuleScore[]) { const score = scores.find((entry) => entry.module === id); if (!score) { - return { id, score: null, killed: null, survived: null, total: null, timeout: null }; + return { + id, + score: null, + killed: null, + survived: null, + total: null, + timeout: null, + noCoverage: null, + errors: null, + }; } const { module: _module, surviving: _surviving, ...measured } = score; return { id, ...measured }; @@ -342,7 +332,11 @@ function readReport(file: string): StrykerReport { * expected-count verdict is the caller's, so a short set is still scored and * published before it fails. */ -function readShardedReports(dir: string): { report: StrykerReport; count: number } { +function readShardedReports(dir: string): { + report: StrykerReport; + count: number; + failed: string[]; +} { const root = path.isAbsolute(dir) ? dir : path.join(repoRoot, dir); // Shard artifacts also carry the lane envelope and the derived test scope, so // the report is selected by name rather than by "every .json here". @@ -352,7 +346,21 @@ function readShardedReports(dir: string): { report: StrykerReport; count: number .sort(); if (files.length === 0) throw new Error(`No Stryker JSON reports found under ${dir}`); process.stdout.write(`mutation: merging ${files.length} shard report(s) from ${dir}\n`); - return { report: mergeReports(files.map(readReport)), count: files.length }; + const shardDirectories = new Set([ + ...files.map((file) => path.dirname(file)), + ...fs + .globSync('**/lane-envelope.json', { cwd: root }) + .map((file) => path.dirname(path.join(root, file))), + ]); + const failed = [...shardDirectories] + .filter( + (directory) => + existingResult(path.join(directory, 'lane-envelope.json')) !== 'pass' || + !fs.existsSync(path.join(directory, path.basename(DEFAULT_REPORT_PATH))), + ) + .map((directory) => path.basename(directory)) + .sort(); + return { report: mergeReports(files.map(readReport)), count: files.length, failed }; } async function produceReport( @@ -364,28 +372,28 @@ async function produceReport( return readReport(reportPath ?? DEFAULT_REPORT_PATH); } -/** - * A merged shard set is only a sweep if every expected shard reported and every - * requested module has mutants — sub-sharded modules make the second check too - * weak alone, since a surviving slice still covers its module. `summarizeReport` - * scores an absent module as 0, so without this a dead shard would be published - * as a `complete`/`pass` envelope claiming the sweep happened. - * - * Both verdicts run after the table is published: the kernels that did complete - * are the part of the run still worth reading on a failed-shard day. - */ -function assertCompleteSweep(state: LaneState, dir: string, shards: number, args: Args): void { - if (args.expectShards !== undefined && shards !== args.expectShards) { - throw new Error( - `Incomplete shard set from ${dir}: ${shards} report(s), expected ${args.expectShards}. ` + - 'A shard job failed or its artifact is absent; the aggregate is not a sweep.', +type ShardSummary = { count: number; failed: readonly string[] }; + +/** Every reported shard must pass; expected counts and module coverage also apply. */ +function incompleteSweepReason( + scores: readonly ModuleScore[], + shards: ShardSummary, + args: Args, +): string | undefined { + if (args.expectShards !== undefined && shards.count !== args.expectShards) { + return ( + `Incomplete shard set from ${args.reportDir}: ${shards.count} report(s), expected ${args.expectShards}. ` + + 'A shard job failed or its artifact is absent; the aggregate is not a sweep.' ); } - const missing = state.scores.filter((score) => score.total === 0).map((score) => score.module); - if (missing.length === 0) return; - throw new Error( - `Incomplete shard set from ${dir}: no mutants for ${missing.join(', ')}. ` + - 'A shard job failed or its artifact is absent; the aggregate is not a sweep.', + if (shards.failed.length > 0) { + return `Shard failure or missing envelope: ${shards.failed.join(', ')}`; + } + const missing = scores.filter((score) => score.total === 0).map((score) => score.module); + if (missing.length === 0) return undefined; + return ( + `Incomplete shard set from ${args.reportDir}: no mutants for ${missing.join(', ')}. ` + + 'A shard job failed or its artifact is absent; the aggregate is not a sweep.' ); } @@ -393,14 +401,14 @@ function assertCompleteSweep(state: LaneState, dir: string, shards: number, args * Reads shard reports, an existing report, or runs Stryker — and scores them. * Returns the shard count when the source was a shard directory. */ -async function scoreModules(args: Args, state: LaneState): Promise { +async function scoreModules(args: Args, state: LaneState): Promise { state.stage = args.reportDir || args.report ? 'report' : 'stryker'; const shards = args.reportDir ? readShardedReports(args.reportDir) : undefined; const report = shards?.report ?? (await produceReport(state.modules, args.report, args.shard)); state.stage = 'score'; state.scores = summarizeReport(report, state.modules); - return shards?.count; + return shards; } async function sweep(args: Args, state: LaneState): Promise { @@ -421,9 +429,9 @@ async function sweep(args: Args, state: LaneState): Promise { const title = args.affected ? 'Mutation score — affected decision kernels' : 'Mutation score — decision kernels'; - emit(renderReport(state.scores, state.provenance, { title }), args.summary); - - if (args.reportDir) assertCompleteSweep(state, args.reportDir, shards ?? 0, args); + const reason = shards ? incompleteSweepReason(state.scores, shards, args) : undefined; + emit(renderReport(state.scores, state.provenance, { title, incomplete: reason }), args.summary); + if (reason) throw new Error(reason); state.stage = 'complete'; return 0; } @@ -435,14 +443,25 @@ async function sweep(args: Args, state: LaneState): Promise { * `workflow.test.ts`, since a filter that missed one would let a harness change * merge unproven, and any wider filter only buys a no-op job. */ -export const LANE_TOOLING = ['scripts/mutation/', 'scripts/lib/', 'stryker.config.json']; +export const LANE_TOOLING = [ + 'scripts/mutation/', + 'scripts/lib/', + 'stryker.config.json', + 'vitest.mutation.config.ts', + 'vitest.config.ts', + '.github/workflows/mutation-weekly.yml', + '.github/workflows/mutation-affected.yml', + 'package.json', + 'pnpm-lock.yaml', + 'packages/*/package.json', +]; /** * The PR lane's matrix: empty unless the diff touches the harness, otherwise the * canary plus whatever kernels that same diff derives. The kernel report that * pays is the weekly sweep, so the PR lane spends mutants on one thing only — * proving the harness still runs end to end when the harness changes. Selecting - * on derived kernel ownership alone would run the full ten-shard sweep on 24 of + * on derived kernel ownership alone would run the full sweep on 24 of * the last 40 merged PRs, a per-PR full sweep for a report nobody gates on. */ export function affectedMatrixFor( @@ -450,7 +469,9 @@ export function affectedMatrixFor( root: string = repoRoot, ): ShardSpec[] { const touchesLane = changed.some((file) => - LANE_TOOLING.some((prefix) => normalizePath(file).startsWith(prefix)), + LANE_TOOLING.some((pattern) => + path.matchesGlob(normalizePath(file), pattern.endsWith('/') ? `${pattern}**` : pattern), + ), ); if (!touchesLane) return []; const modules = new Set(derivedAffectedModules(changed, root)); diff --git a/scripts/mutation/score.ts b/scripts/mutation/score.ts index 57bb7efbbe..8552d2fa39 100644 --- a/scripts/mutation/score.ts +++ b/scripts/mutation/score.ts @@ -17,6 +17,7 @@ export type SurvivingMutant = { readonly file: string; readonly line: number; readonly mutator: string; + readonly status: string; }; export type ModuleScore = { @@ -28,6 +29,8 @@ export type ModuleScore = { /** Timeouts, already counted in `killed` — reported so a score propped up by * slow mutants rather than assertions is visible. */ readonly timeout: number; + readonly noCoverage: number; + readonly errors: number; readonly surviving: readonly SurvivingMutant[]; }; @@ -63,7 +66,14 @@ function compareMutants(a: SurvivingMutant, b: SurvivingMutant): number { return a.file.localeCompare(b.file) || a.line - b.line || a.mutator.localeCompare(b.mutator); } -type Bucket = { killed: number; survived: number; timeout: number; surviving: SurvivingMutant[] }; +type Bucket = { + killed: number; + survived: number; + timeout: number; + noCoverage: number; + errors: number; + surviving: SurvivingMutant[]; +}; function tally(bucket: Bucket, file: string, mutant: StrykerMutant): void { if (KILLED_STATUSES.has(mutant.status)) { @@ -71,12 +81,15 @@ function tally(bucket: Bucket, file: string, mutant: StrykerMutant): void { if (mutant.status === 'Timeout') bucket.timeout += 1; return; } + if (mutant.status === 'CompileError' || mutant.status === 'RuntimeError') bucket.errors += 1; if (!SURVIVED_STATUSES.has(mutant.status)) return; + if (mutant.status === 'NoCoverage') bucket.noCoverage += 1; bucket.survived += 1; bucket.surviving.push({ file: normalizePath(file), line: mutant.location?.start?.line ?? 0, mutator: mutant.mutatorName ?? 'unknown', + status: mutant.status, }); } @@ -85,7 +98,15 @@ export function summarizeReport( ids: readonly ModuleId[] = ALL_MODULE_IDS, ): ModuleScore[] { const buckets = new Map(); - for (const id of ids) buckets.set(id, { killed: 0, survived: 0, timeout: 0, surviving: [] }); + for (const id of ids) + buckets.set(id, { + killed: 0, + survived: 0, + timeout: 0, + noCoverage: 0, + errors: 0, + surviving: [], + }); for (const [file, entry] of Object.entries(report.files)) { const id = moduleForFile(file); @@ -103,6 +124,8 @@ export function summarizeReport( survived: bucket.survived, total, timeout: bucket.timeout, + noCoverage: bucket.noCoverage, + errors: bucket.errors, surviving: bucket.surviving.sort(compareMutants), }; }); diff --git a/scripts/mutation/selection.test.ts b/scripts/mutation/selection.test.ts index 95b602a7e1..1725105b25 100644 --- a/scripts/mutation/selection.test.ts +++ b/scripts/mutation/selection.test.ts @@ -90,6 +90,27 @@ test('a kernel diff selects nothing — only a harness diff spends mutants', () ); }); -test('a docs-only diff selects nothing', () => { - assert.deepEqual(affectedMatrixFor(['docs/agents/testing.md']), []); +test('non-harness paths, including near-miss manifests, select nothing', () => { + for (const file of [ + 'docs/agents/testing.md', + 'package.json.md', + 'packages/kernel/src/package.json', + 'packages/selectors/src/internal/parse.ts', + ]) { + assert.deepEqual(affectedMatrixFor([file]), [], file); + } +}); + +test('workflow, Vitest, dependency, and workspace manifest changes select the harness canary', () => { + for (const file of [ + 'vitest.mutation.config.ts', + 'vitest.config.ts', + '.github/workflows/mutation-weekly.yml', + '.github/workflows/mutation-affected.yml', + 'package.json', + 'pnpm-lock.yaml', + 'packages/kernel/package.json', + ]) { + assert.deepEqual(affectedMatrixFor([file]), shardMatrix([LANE_CANARY]), file); + } }); diff --git a/scripts/mutation/sharding.test.ts b/scripts/mutation/sharding.test.ts new file mode 100644 index 0000000000..2774a87786 --- /dev/null +++ b/scripts/mutation/sharding.test.ts @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; +import path from 'node:path'; +import { test } from 'node:test'; +import { parseSync } from 'oxc-parser'; +import { shardFiles } from './sharding.ts'; + +test('type-only syntax cannot outweigh regex mutations when packing shards', async () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-regex-density-')); + try { + fs.writeFileSync( + path.join(root, 'types.ts'), + `export type Fields = {\n${Array.from({ length: 500 }, (_, index) => ` key${index}: 'value${index}';\n`).join('')}};\n`, + ); + fs.writeFileSync( + path.join(root, 'patterns.ts'), + 'export const first = /^(a|b)?[0-9]+$/;\nexport const second = /^(c|d)?[a-z]+$/;\n', + ); + for (const index of [1, 2]) { + const selected = await shardFiles(['types.ts', 'patterns.ts'], { index, count: 2 }, root); + assert.ok(selected.some((range) => range.startsWith('patterns.ts:'))); + assert.ok(selected.every((range) => !range.startsWith('types.ts'))); + } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('compact decisions are spread across shards instead of outweighed by comments', async () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-density-')); + try { + fs.writeFileSync( + path.join(root, 'verbose.ts'), + `${'// long explanation\n'.repeat(500)}export const fallback = false;\n`, + ); + fs.writeFileSync( + path.join(root, 'dense.ts'), + Array.from( + { length: 8 }, + (_, index) => + `export function f${index}(x: number) { return x > 0 && x < 10 ? x + 1 : x - 1; }\n`, + ).join(''), + ); + for (const index of [1, 2]) { + assert.ok( + (await shardFiles(['verbose.ts', 'dense.ts'], { index, count: 2 }, root)).some((range) => + range.startsWith('dense.ts:'), + ), + `shard ${index} has no compact decisions`, + ); + } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('a single-file kernel is split without losing or duplicating statement-spanning mutants', async () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-sharding-')); + const source = `export const label = "🧪";\nexport function first() {\n${Array.from({ length: 15 }, (_, index) => ` const a${index} = 1;\n`).join('')} return true;\n}\nexport const second = false;\nexport function third() {\n${Array.from({ length: 15 }, (_, index) => ` const b${index} = 2;\n`).join('')} return false;\n}\n`; + try { + fs.writeFileSync(path.join(root, 'kernel.ts'), source); + const ranges = ( + await Promise.all([1, 2].map((index) => shardFiles(['kernel.ts'], { index, count: 2 }, root))) + ).flat(); + assert.ok(ranges.length >= 2); + const offset = (line: number, column: number) => + source + .split('\n') + .slice(0, line - 1) + .reduce((sum, text) => sum + text.length + 1, 0) + column; + const spans = ranges + .map((range) => { + const match = /:(\d+):(\d+)-(\d+):(\d+)$/.exec(range)!; + return { + start: offset(Number(match[1]), Number(match[2])), + end: offset(Number(match[3]), Number(match[4])), + }; + }) + .sort((a, b) => a.start - b.start); + assert.equal(spans[0]!.start, 0); + for (let index = 1; index < spans.length; index++) { + assert.equal(spans[index - 1]!.end, spans[index]!.start); + } + assert.equal(spans.at(-1)!.end, source.length); + for (const statement of parseSync('kernel.ts', source).program.body) { + assert.equal( + spans.filter((range) => range.start <= statement.start && range.end >= statement.end) + .length, + 1, + ); + } + assert.notDeepEqual( + await shardFiles(['kernel.ts'], { index: 1, count: 2 }, root), + await shardFiles(['kernel.ts'], { index: 2, count: 2 }, root), + ); + await assert.rejects( + () => shardFiles(['kernel.ts'], { index: 8, count: 8 }, root), + /no sources/, + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/mutation/sharding.ts b/scripts/mutation/sharding.ts new file mode 100644 index 0000000000..bd493adae1 --- /dev/null +++ b/scripts/mutation/sharding.ts @@ -0,0 +1,122 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { parseSync } from 'oxc-parser'; +import { Instrumenter } from '@stryker-mutator/instrumenter'; + +export type Shard = { index: number; count: number }; + +type Slice = { file: string; start: number; end: number; source: string; weight: number }; + +const silent = () => {}; +const disabled = () => false; +const enabled = () => true; +const instrumenter = new Instrumenter({ + trace: silent, + debug: silent, + info: silent, + warn: console.warn, + error: console.error, + fatal: console.error, + isTraceEnabled: disabled, + isDebugEnabled: disabled, + isInfoEnabled: disabled, + isWarnEnabled: enabled, + isErrorEnabled: enabled, + isFatalEnabled: enabled, +}); + +function position(source: string, offset: number): string { + const prefix = source.slice(0, offset); + const line = prefix.split('\n').length; + return `${line}:${offset - (prefix.lastIndexOf('\n') + 1)}`; +} + +/** Partition at top-level statement boundaries so no mutant straddles a cut. */ +function slices( + file: string, + source: string, + statements: readonly { start: number; weight: number }[], + budget: number, +): Slice[] { + const result: Slice[] = []; + let start = 0; + let weight = 0; + for (const statement of statements) { + if (weight > 0 && weight + statement.weight > budget) { + result.push({ file, start, end: statement.start, source, weight }); + start = statement.start; + weight = 0; + } + weight += statement.weight; + } + result.push({ file, start, end: source.length, source, weight }); + return result; +} + +/** Balance Stryker-generated mutations while retaining complete top-level statements. */ +export async function shardFiles( + files: readonly string[], + shard: Shard, + root: string, + excludedMutations: readonly string[] = [], +): Promise { + const sources = files.map((file) => { + const source = fs.readFileSync(path.join(root, file), 'utf8'); + const parsed = parseSync(file, source); + if (parsed.errors.length > 0) + throw new Error(`Cannot shard ${file}: ${parsed.errors[0]!.message}`); + return { file, source, statements: parsed.program.body }; + }); + const generated = await instrumenter.instrument( + sources.map(({ file, source }) => ({ + name: path.resolve(root, file), + content: source, + mutate: true, + })), + { excludedMutations: [...excludedMutations], ignorers: [] }, + ); + // Two groups per job give the packer room to spread mutation-dense files. + const budget = Math.max(1, Math.ceil(generated.mutants.length / (shard.count * 2))); + const weighed = sources + .flatMap(({ file, source, statements }) => { + const starts: number[] = []; + let cursor = 0; + for (const line of source.split('\n')) { + starts.push(cursor); + cursor += line.length + 1; + } + const offset = (location: { line: number; column: number }) => + starts[location.line]! + location.column; + const mutants = generated.mutants.filter( + (mutant) => mutant.fileName === path.resolve(root, file), + ); + const weighted = statements.map((statement) => ({ + start: statement.start, + weight: mutants.filter( + (mutant) => + offset(mutant.location.start) >= statement.start && + offset(mutant.location.end) <= statement.end, + ).length, + })); + if (weighted.reduce((sum, statement) => sum + statement.weight, 0) !== mutants.length) + throw new Error(`Cannot shard ${file}: a mutation spans top-level statements`); + return slices(file, source, weighted, budget).filter((slice) => slice.weight > 0); + }) + .sort((a, b) => b.weight - a.weight || a.file.localeCompare(b.file) || a.start - b.start); + const bins = Array.from({ length: shard.count }, () => ({ size: 0, slices: [] as Slice[] })); + for (const slice of weighed) { + const bin = bins.reduce((smallest, next) => (next.size < smallest.size ? next : smallest)); + bin.slices.push(slice); + bin.size += slice.weight; + } + const selected = bins[shard.index - 1]!.slices; + if (selected.length === 0) + throw new Error(`Mutation shard ${shard.index}/${shard.count} has no sources`); + return selected + .map(({ file, source, start, end }) => + start === 0 && end === source.length + ? file + : `${file}:${position(source, start)}-${position(source, end)}`, + ) + .sort(); +} diff --git a/scripts/mutation/test-scope.test.ts b/scripts/mutation/test-scope.test.ts index c29145b6af..43d61ba3b8 100644 --- a/scripts/mutation/test-scope.test.ts +++ b/scripts/mutation/test-scope.test.ts @@ -1,7 +1,15 @@ import assert from 'node:assert/strict'; import path from 'node:path'; +import fs from 'node:fs'; +import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; import { test } from 'node:test'; -import { expandMutateFiles, threadHostileTestFiles } from './test-scope.ts'; +import { + expandMutateFiles, + relatedTestFiles, + readTestScope, + TEST_SCOPE_ENV, + threadHostileTestFiles, +} from './test-scope.ts'; import { mutateGlobs } from './modules.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); @@ -26,3 +34,57 @@ test('the thread-hostile exclusion is derived from imports, not listed', () => { assert.ok(!files.includes('src/daemon/__tests__/ref-frame.test.ts')); assert.ok(files.every((file) => file.endsWith('.test.ts'))); }); + +test('related scope discovery follows indirect imports without executing tests', async () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-scope-')); + try { + fs.mkdirSync(path.join(root, 'src')); + fs.writeFileSync( + path.join(root, 'package.json'), + JSON.stringify({ name: 'scope-fixture', type: 'module' }), + ); + fs.symlinkSync(path.join(repoRoot, 'node_modules'), path.join(root, 'node_modules')); + const marker = path.join(root, 'executed'); + fs.writeFileSync( + path.join(root, 'vitest.config.ts'), + `export default { test: { projects: [{ test: { name: 'unit-core', include: ['src/**/*.test.ts'] } }] } };`, + ); + fs.writeFileSync(path.join(root, 'src/kernel.ts'), 'export const value = 1;'); + fs.writeFileSync(path.join(root, 'src/consumer.ts'), "export { value } from './kernel';"); + fs.writeFileSync( + path.join(root, 'src/consumer.test.ts'), + `import { value } from './consumer'; import { writeFileSync } from 'node:fs'; writeFileSync(${JSON.stringify(marker)}, String(value)); throw new Error('discovery must not execute tests');`, + ); + fs.writeFileSync(path.join(root, 'src/unrelated.test.ts'), "throw new Error('unrelated');"); + assert.deepEqual(await relatedTestFiles(['src/kernel.ts'], root, []), ['src/consumer.test.ts']); + assert.equal(fs.existsSync(marker), false, 'scope discovery executed the test module'); + assert.deepEqual(await relatedTestFiles(['src/kernel.ts'], root, ['src/consumer.test.ts']), []); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('a missing scope file is rejected and an empty scope file reads as an empty list', () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-empty-scope-')); + const previous = process.env[TEST_SCOPE_ENV]; + try { + const file = path.join(root, 'scope.json'); + process.env[TEST_SCOPE_ENV] = file; + assert.throws(() => readTestScope(), /scope/i); + fs.writeFileSync(file, '[]'); + assert.deepEqual(readTestScope(), []); + } finally { + if (previous === undefined) delete process.env[TEST_SCOPE_ENV]; + else process.env[TEST_SCOPE_ENV] = previous; + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('the selector mutation scope measures production decisions rather than test fixture builders', () => { + const files = expandMutateFiles(mutateGlobs(['selectors']), repoRoot); + assert.equal( + files.some((file) => file.endsWith('.fixtures.ts')), + false, + ); + assert.ok(files.includes('packages/selectors/src/internal/parse.ts')); +}); diff --git a/scripts/mutation/test-scope.ts b/scripts/mutation/test-scope.ts index c0d99d1ba4..4127468f29 100644 --- a/scripts/mutation/test-scope.ts +++ b/scripts/mutation/test-scope.ts @@ -2,10 +2,10 @@ // // Stryker replays the configured suite for every mutant, so the suite it is // pointed at decides whether the weekly sweep fits its 30-minute budget. Pointing -// it at the whole unit suite (487 files) makes the initial dry run alone cost +// it at the whole unit suite makes the initial dry run alone cost // minutes; hand-listing per-kernel test files would be a second source of truth // that silently rots. So the scope is derived the same way `pnpm check:affected` -// derives affected tests: `vitest related` over the mutated files, i.e. Vitest's +// derives affected tests: Vitest related-file discovery over the mutated files, i.e. Vitest's // own static module graph. // // Two files are removed from whatever Vitest returns: @@ -27,7 +27,6 @@ import fs from 'node:fs'; import path from 'node:path'; -import { runCmdSync } from '@agent-device/host-kit/command'; import { readWorkspacePackages } from '../layering/package-boundaries.ts'; import { walkFiles } from '../lib/walk-files.ts'; import { isKernelTestFile, normalizePath } from './modules.ts'; @@ -94,58 +93,45 @@ export function expandMutateFiles(globs: readonly string[], repoRoot: string): s .sort(); } -type VitestJsonReport = { testResults?: readonly { name: string }[] }; - -/** - * Test files Vitest considers related to `sourceFiles`, minus the groups this - * lane cannot run. `vitest related` executes them once (seconds), which also - * proves the scope is green before Stryker's dry run depends on it. - */ -export function relatedTestFiles( +/** Discover Vitest's related files without executing a preflight test suite. */ +export async function relatedTestFiles( sourceFiles: readonly string[], repoRoot: string, excluded: readonly string[] = threadHostileTestFiles(repoRoot), -): string[] { - const reportFile = path.join(repoRoot, '.tmp/mutation/related-tests.json'); - fs.mkdirSync(path.dirname(reportFile), { recursive: true }); - fs.rmSync(reportFile, { force: true }); - runCmdSync( - 'pnpm', - [ - 'exec', - 'vitest', - 'related', - ...sourceFiles, - '--project', - 'unit-core', - '--run', - '--reporter=json', - `--outputFile=${reportFile}`, - ], - { cwd: repoRoot, allowFailure: true }, - ); - if (!fs.existsSync(reportFile)) { - throw new Error( - `vitest related produced no report at ${reportFile} — cannot derive the mutation test scope.`, - ); +): Promise { + const { createVitest } = await import('vitest/node'); + const vitest = await createVitest('test', { + root: repoRoot, + config: path.join(repoRoot, 'vitest.config.ts'), + project: ['unit-core'], + related: sourceFiles.map((file) => path.resolve(repoRoot, file)), + watch: false, + }); + try { + const specs = await vitest.getRelevantTestSpecifications(); + const excludedSet = new Set(excluded); + return [ + ...new Set( + specs + .map((spec) => normalizePath(path.relative(repoRoot, spec.moduleId))) + .filter((file) => isKernelTestFile(file) && !excludedSet.has(file)), + ), + ].sort(); + } finally { + await vitest.close(); } - const report = JSON.parse(fs.readFileSync(reportFile, 'utf8')) as VitestJsonReport; - const excludedSet = new Set(excluded); - return [ - ...new Set( - (report.testResults ?? []) - .map((result) => normalizePath(path.relative(repoRoot, result.name))) - .filter((file) => isKernelTestFile(file) && !excludedSet.has(file)), - ), - ].sort(); } /** Read the scope Stryker was handed, or `undefined` for "whole unit suite". */ export function readTestScope(): string[] | undefined { const file = process.env[TEST_SCOPE_ENV]; - if (!file || !fs.existsSync(file)) return undefined; + if (!file) return undefined; + if (!fs.existsSync(file)) throw new Error(`Mutation test scope is missing: ${file}`); const files = JSON.parse(fs.readFileSync(file, 'utf8')) as string[]; - return files.length > 0 ? files : undefined; + if (!Array.isArray(files) || files.some((entry) => typeof entry !== 'string')) { + throw new Error(`Invalid mutation test scope: ${file}`); + } + return files; } export function writeTestScope(files: readonly string[], file: string): void { diff --git a/scripts/mutation/workflow.test.ts b/scripts/mutation/workflow.test.ts index 0a0e10e268..5ce7774022 100644 --- a/scripts/mutation/workflow.test.ts +++ b/scripts/mutation/workflow.test.ts @@ -8,6 +8,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'node:test'; +import { parse } from 'yaml'; import { shardMatrix } from './modules.ts'; import { LANE_TOOLING } from './run.ts'; @@ -67,11 +68,7 @@ test('the affected lane triggers on exactly the lane sources that can select mut const paths = [ ...workflow('mutation-affected.yml').matchAll(/^ {6}- (?['"])(?[^'"]+)\k$/gm), ].map((match) => match.groups!.glob); - const expected = [ - ...LANE_TOOLING.map((prefix) => (prefix.endsWith('/') ? `${prefix}**` : prefix)), - // The workflow reruns itself so a trigger edit is proven by the lane it edits. - '.github/workflows/mutation-affected.yml', - ]; + const expected = LANE_TOOLING.map((prefix) => (prefix.endsWith('/') ? `${prefix}**` : prefix)); assert.deepEqual( [...paths].sort(), [...expected].sort(), @@ -79,3 +76,33 @@ test('the affected lane triggers on exactly the lane sources that can select mut ); assert.match(workflow('mutation-affected.yml'), /gate: mutation-affected[\s\S]*--list-affected/); }); + +test('mutant job budgets include setup and unconditional evidence upload', () => { + for (const name of ['mutation-weekly.yml', 'mutation-affected.yml']) { + const jobs = parse(workflow(name)).jobs; + const job = jobs[name === 'mutation-weekly.yml' ? 'shard' : 'mutants']; + assert.equal(job['timeout-minutes'], 30, name); + const steps = job.steps as { name: string; 'timeout-minutes'?: number; if?: string }[]; + const mutantIndex = steps.findIndex((step) => step.name.startsWith('Run mutants for')); + assert.ok(mutantIndex > 0, name); + const setup = steps.slice(0, mutantIndex); + for (const step of setup) { + assert.ok( + Number.isInteger(step['timeout-minutes']) && step['timeout-minutes']! > 0, + `${name}: unbounded setup step ${step.name}`, + ); + } + assert.equal(steps[mutantIndex]!['timeout-minutes'], 22, name); + const setupBudget = setup.reduce((total, step) => total + step['timeout-minutes']!, 0); + assert.ok( + job['timeout-minutes'] - setupBudget - steps[mutantIndex]!['timeout-minutes']! >= 5, + `${name}: setup and mutants consume the evidence/upload budget`, + ); + assert.equal( + steps.find((step) => step.name === 'Record a failed shard envelope')?.if, + 'failure() || cancelled()', + name, + ); + assert.equal(steps.find((step) => step.name === 'Upload shard report')?.if, 'always()', name); + } +}); diff --git a/stryker.config.json b/stryker.config.json index f8154a4c27..e019504f93 100644 --- a/stryker.config.json +++ b/stryker.config.json @@ -7,17 +7,19 @@ "plugins_comment": "Named explicitly: the default `@stryker-mutator/*` glob does not resolve under pnpm's non-flat node_modules, so plugin discovery finds no test runner.", "plugins": ["@stryker-mutator/vitest-runner"], "testRunner": "vitest", + "vitest_comment": "The harness derives the related test scope once. Per-test coverage filters each mutant; repeating Vitest related discovery during every mutant adds no coverage.", "vitest": { "configFile": "vitest.mutation.config.ts", - "related": true + "related": false }, - "reporters": ["progress", "clear-text", "json"], + "reporters": ["progress", "json"], "jsonReporter": { "fileName": ".tmp/mutation/mutation.json" }, "htmlReporter": { "fileName": ".tmp/mutation/mutation.html" }, + "concurrency": 4, "coverageAnalysis": "perTest", "ignoreStatic": true, "tempDirName": ".tmp/stryker", @@ -38,6 +40,7 @@ "packages/capture-kit/src/snapshot/scroll-edge-state/**/*.ts", "packages/selectors/src/**/*.ts", "!packages/selectors/src/**/*.test.ts", + "!packages/selectors/src/**/*.fixtures.ts", "!packages/selectors/src/internal/__tests__/**", "packages/ad-script/src/internal/target-annotation-serde.ts", "packages/capture-kit/src/snapshot-occlusion.ts" diff --git a/vitest.mutation.config.ts b/vitest.mutation.config.ts index 24ab16fb98..ef44132872 100644 --- a/vitest.mutation.config.ts +++ b/vitest.mutation.config.ts @@ -14,13 +14,15 @@ const workspaceAliases = workspaceSourceAliases( ); const scope = readTestScope(); +if (scope === undefined) + throw new Error('Mutation test scope must be supplied by scripts/mutation/run.ts'); export default defineConfig({ resolve: { alias: workspaceAliases, }, test: { - include: scope ?? ['src/**/*.test.ts', 'packages/*/src/**/*.test.ts'], + include: scope, exclude: [ ...MUTATION_EXCLUDED_TESTS, ...threadHostileTestFiles(repoRoot),