From aea8c1c1cc88e8023f432b29d7636b8681775026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 16:20:51 +0200 Subject: [PATCH 1/6] perf: add opt-in fresh mutation coverage scope pilot --- docs/agents/testing.md | 4 +- scripts/mutation/coverage-reporter.ts | 15 ++ scripts/mutation/coverage-scope.test.ts | 144 ++++++++++++++ scripts/mutation/coverage-scope.ts | 165 ++++++++++++++++ scripts/mutation/envelope.test.ts | 33 ++++ scripts/mutation/run.ts | 249 ++++++++++++++++++++++-- scripts/mutation/seed-setup.ts | 7 + scripts/mutation/seed.test.ts | 31 +++ scripts/mutation/seed.ts | 9 + vitest.mutation.config.ts | 2 +- 10 files changed, 637 insertions(+), 22 deletions(-) create mode 100644 scripts/mutation/coverage-reporter.ts create mode 100644 scripts/mutation/coverage-scope.test.ts create mode 100644 scripts/mutation/coverage-scope.ts create mode 100644 scripts/mutation/seed-setup.ts create mode 100644 scripts/mutation/seed.test.ts create mode 100644 scripts/mutation/seed.ts diff --git a/docs/agents/testing.md b/docs/agents/testing.md index ce43016caa..f7ac40e1de 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -125,8 +125,8 @@ worker process, so a case that never returns or kills its process is reported as against the exact input. Promote a discovered case with the command the harness prints — never hand-copy an unshrunk input. -Scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` (PR canary). -Scores are advisory; incomplete sweeps fail. Coverage does not prove test strength. +Mutation scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` +(PR canary). Scores are advisory; incomplete sweeps fail. Pilot protocol: `pnpm mutation:run --help`. ## Before editing a shared module diff --git a/scripts/mutation/coverage-reporter.ts b/scripts/mutation/coverage-reporter.ts new file mode 100644 index 0000000000..0e98926223 --- /dev/null +++ b/scripts/mutation/coverage-reporter.ts @@ -0,0 +1,15 @@ +import fs from 'node:fs'; +import { declareValuePlugin, PluginKind } from '@stryker-mutator/api/plugin'; +import type { Reporter } from '@stryker-mutator/api/report'; + +export const COVERAGE_CAPTURE_ENV = 'AGENT_DEVICE_MUTATION_COVERAGE_CAPTURE'; + +const reporter: Reporter = { + onDryRunCompleted(event) { + const target = process.env[COVERAGE_CAPTURE_ENV]; + if (!target) throw new Error('Mutation coverage capture path is missing'); + fs.writeFileSync(target, JSON.stringify(event)); + }, +}; + +export const strykerPlugins = [declareValuePlugin(PluginKind.Reporter, 'scope-coverage', reporter)]; diff --git a/scripts/mutation/coverage-scope.test.ts b/scripts/mutation/coverage-scope.test.ts new file mode 100644 index 0000000000..cff597dafc --- /dev/null +++ b/scripts/mutation/coverage-scope.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'node:test'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import type { DryRunCompletedEvent } from '@stryker-mutator/api/report'; +import { DryRunStatus, TestStatus } from '@stryker-mutator/api/test-runner'; +import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts'; +import { + assertCoveragePolicy, + coverageScope, + scopedTestFiles, + scopeFingerprint, +} from './coverage-scope.ts'; + +const root = path.resolve(import.meta.dirname, '../..'); +const identity = { + module: 'snapshot-occlusion' as const, + epoch: 'invocation-1', + seed: 123, + fingerprint: 'inputs-1', +}; +const candidates = ['src/a.test.ts', 'src/b.test.ts', 'src/unused.test.ts']; +function event(): DryRunCompletedEvent { + return { + result: { + status: DryRunStatus.Complete, + tests: candidates.map((file, index) => ({ + id: String(index), + name: file, + fileName: path.join(root, file), + status: TestStatus.Success, + timeSpentMs: 1, + })), + mutantCoverage: { + static: { declaration: 1 }, + perTest: { '0': { alpha: 1 }, '1': { beta: 1 }, '2': { alpha: 0 } }, + }, + }, + timing: { net: 9, overhead: 1 }, + capabilities: { reloadEnvironment: true }, + }; +} + +test('coverage union keeps indirect covering tests, omits zero hits and intersects shard candidates', () => { + const scope = coverageScope(event(), identity, candidates, root, 20); + assert.deepEqual(scope.covered, ['src/a.test.ts', 'src/b.test.ts']); + assert.equal(scope.initialRunMs, 10); + assert.equal(scope.producerMs, 20); + assert.deepEqual(scopedTestFiles(scope, identity, ['src/b.test.ts', 'src/unused.test.ts']), [ + 'src/b.test.ts', + ]); + assert.throws( + () => scopedTestFiles(scope, identity, ['src/unused.test.ts']), + /selected no tests/, + ); +}); + +test('missing coverage, failed tests and unattributed covering tests cannot produce an index', () => { + const missing = event(); + delete missing.result.mutantCoverage; + assert.throws(() => coverageScope(missing, identity, candidates, root, 20), /per-test/); + const failed = event(); + failed.result.tests[0] = { + ...failed.result.tests[0]!, + status: TestStatus.Failed, + failureMessage: 'broken', + }; + assert.throws(() => coverageScope(failed, identity, candidates, root, 20), /failed tests/); + const unlocated = event(); + delete unlocated.result.tests[0]!.fileName; + assert.throws(() => coverageScope(unlocated, identity, candidates, root, 20), /cannot attribute/); + const outside = event(); + outside.result.tests[0]!.fileName = path.join(root, 'src/other.test.ts'); + assert.throws(() => coverageScope(outside, identity, candidates, root, 20), /unexpected test/); +}); + +test('identity drift, new candidates and a truncated or corrupted index are rejected', () => { + const scope = coverageScope(event(), identity, candidates, root, 20); + for (const expected of [ + { ...identity, seed: 124 }, + { ...identity, epoch: 'other' }, + { ...identity, fingerprint: 'changed' }, + { ...identity, module: 'selectors' as const }, + ]) + assert.throws(() => scopedTestFiles(scope, expected, candidates), /does not match/); + assert.throws( + () => scopedTestFiles(scope, identity, [...candidates, 'src/new.test.ts']), + /omits/, + ); + assert.throws( + () => scopedTestFiles({ ...scope, covered: ['src/a.test.ts'] }, identity, candidates), + /checksum/, + ); + assert.throws( + () => scopedTestFiles({ ...scope, covered: [] }, identity, candidates), + /invalid test/, + ); +}); + +test('pilot policy rejects static audits and incremental reuse', () => { + assertCoveragePolicy({ coverageAnalysis: 'perTest', ignoreStatic: true }); + for (const config of [ + { coverageAnalysis: 'all', ignoreStatic: true }, + { coverageAnalysis: 'perTest', ignoreStatic: false }, + { coverageAnalysis: 'perTest', ignoreStatic: true, incremental: true }, + ]) + assert.throws(() => assertCoveragePolicy(config), /fresh perTest/); +}); + +test('fingerprint invalidates source helpers, fixtures, setup, configs, new/deleted tests and environment', () => { + const fixture = fs.realpathSync(mkdtempForTestSync('mutation-inputs-')); + runCmdSync('git', ['init', '-q', fixture]); + fs.symlinkSync(path.join(root, 'node_modules'), path.join(fixture, 'node_modules')); + fs.writeFileSync(path.join(fixture, '.gitignore'), 'node_modules\n'); + for (const file of [ + 'helper.ts', + 'fixture.json', + 'setup.ts', + 'stryker.config.json', + 'pnpm-lock.yaml', + 'test.test.ts', + ]) { + const target = path.join(fixture, file); + fs.writeFileSync(target, 'before'); + const before = scopeFingerprint(fixture); + fs.writeFileSync(target, 'after'); + assert.notEqual(scopeFingerprint(fixture), before, file); + } + const before = scopeFingerprint(fixture); + fs.writeFileSync(path.join(fixture, 'new.test.ts'), 'new'); + assert.notEqual(scopeFingerprint(fixture), before); + fs.unlinkSync(path.join(fixture, 'new.test.ts')); + assert.equal(scopeFingerprint(fixture), before); + fs.unlinkSync(path.join(fixture, 'test.test.ts')); + assert.notEqual(scopeFingerprint(fixture), before); + const withoutEnv = scopeFingerprint(fixture); + process.env.MUTATION_SCOPE_INPUT_TEST = 'changed'; + try { + assert.notEqual(scopeFingerprint(fixture), withoutEnv); + } finally { + delete process.env.MUTATION_SCOPE_INPUT_TEST; + } +}); diff --git a/scripts/mutation/coverage-scope.ts b/scripts/mutation/coverage-scope.ts new file mode 100644 index 0000000000..29d4ae95b2 --- /dev/null +++ b/scripts/mutation/coverage-scope.ts @@ -0,0 +1,165 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import type { DryRunCompletedEvent } from '@stryker-mutator/api/report'; +import { DryRunStatus, TestStatus } from '@stryker-mutator/api/test-runner'; +import { normalizePath, type ModuleId } from './modules.ts'; + +export type ScopeIdentity = { module: ModuleId; epoch: string; seed: number; fingerprint: string }; +export type CoverageScope = { + schemaVersion: 1; + identity: ScopeIdentity; + candidates: string[]; + covered: string[]; + initialRunMs: number; + producerMs: number; + checksum: string; +}; + +function scopeChecksum(scope: Omit): string { + return crypto + .createHash('sha256') + .update( + JSON.stringify([ + scope.schemaVersion, + [ + scope.identity.module, + scope.identity.epoch, + scope.identity.seed, + scope.identity.fingerprint, + ], + scope.candidates, + scope.covered, + scope.initialRunMs, + scope.producerMs, + ]), + ) + .digest('hex'); +} + +/** Conservative pilot identity: content/inventory, installed tools, Node and environment. */ +export function scopeFingerprint(root: string): string { + const hash = crypto.createHash('sha256'); + const files = runCmdSync( + 'git', + ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], + { + cwd: root, + }, + ) + .stdout.split('\0') + .filter(Boolean) + .sort(); + for (const file of files) { + hash.update( + JSON.stringify([ + file, + fs.statSync(path.join(root, file)).mode, + crypto + .createHash('sha256') + .update(fs.readFileSync(path.join(root, file))) + .digest('hex'), + ]), + ); + } + for (const name of [ + '@stryker-mutator/core', + '@stryker-mutator/api', + '@stryker-mutator/instrumenter', + '@stryker-mutator/vitest-runner', + 'vitest', + 'fast-check', + ]) { + const pkg = JSON.parse( + fs.readFileSync(path.join(root, 'node_modules', name, 'package.json'), 'utf8'), + ); + hash.update(JSON.stringify([name, pkg.version])); + } + const environment = Object.entries(process.env) + .filter(([key]) => !['_', 'SHLVL', 'PWD', 'OLDPWD'].includes(key)) + .sort(([a], [b]) => a.localeCompare(b)); + hash.update(JSON.stringify([process.versions, process.platform, process.arch, environment])); + return `sha256:${hash.digest('hex')}`; +} + +/** The union Stryker's current per-test planner can actually schedule. */ +export function coverageScope( + event: DryRunCompletedEvent, + identity: ScopeIdentity, + candidates: readonly string[], + root: string, + producerMs: number, +): CoverageScope { + const coverage = event.result.mutantCoverage?.perTest; + if (event.result.status !== DryRunStatus.Complete || !coverage) + throw new Error('Scope probe must provide successful per-test mutation coverage'); + const tests = new Map(event.result.tests.map((test) => [test.id, test])); + if (event.result.tests.some((test) => test.status === TestStatus.Failed)) + throw new Error('Scope probe contains failed tests'); + const allowed = new Set(candidates); + const covered = new Set(); + for (const [id, hits] of Object.entries(coverage)) { + if (!Object.values(hits).some((count) => count > 0)) continue; + const test = tests.get(id); + if (!test?.fileName) throw new Error(`Scope probe cannot attribute covering test ${id}`); + const file = normalizePath(path.relative(root, test.fileName)); + if (!allowed.has(file)) + throw new Error(`Scope probe returned an unexpected test file: ${file}`); + covered.add(file); + } + if (covered.size === 0) throw new Error('Scope probe has no attributed covering test files'); + const scope: Omit = { + schemaVersion: 1, + identity, + candidates: [...candidates].sort(), + covered: [...covered].sort(), + initialRunMs: event.timing.net + event.timing.overhead, + producerMs, + }; + return { ...scope, checksum: scopeChecksum(scope) }; +} + +export function scopedTestFiles( + index: CoverageScope, + expected: ScopeIdentity, + candidates: readonly string[], +): string[] { + if ( + index?.schemaVersion !== 1 || + !index.identity || + (Object.keys(expected) as (keyof ScopeIdentity)[]).some( + (key) => index.identity[key] !== expected[key], + ) + ) + throw new Error('Coverage scope does not match this module, epoch, seed or execution inputs'); + if ( + !Array.isArray(index.candidates) || + index.candidates.some((file) => typeof file !== 'string') || + !Array.isArray(index.covered) || + !index.covered.length || + index.covered.some((file) => typeof file !== 'string' || !index.candidates.includes(file)) + ) + throw new Error('Coverage scope contains invalid test files'); + if ( + index.checksum !== scopeChecksum(index) || + !Number.isFinite(index.producerMs) || + !Number.isFinite(index.initialRunMs) + ) + throw new Error('Coverage scope checksum or timing is invalid'); + if (candidates.some((file) => !index.candidates.includes(file))) + throw new Error('Coverage scope omits a current candidate test file'); + const allowed = new Set(candidates); + const selected = [...new Set(index.covered.filter((file) => allowed.has(file)))].sort(); + if (!selected.length) throw new Error('Coverage scope selected no tests for this shard'); + return selected; +} + +export function assertCoveragePolicy(config: { + coverageAnalysis?: string; + ignoreStatic?: boolean; + incremental?: boolean; +}): void { + if (config.coverageAnalysis !== 'perTest' || config.ignoreStatic !== true || config.incremental) + throw new Error('Coverage scope requires fresh perTest coverage with ignoreStatic enabled'); +} diff --git a/scripts/mutation/envelope.test.ts b/scripts/mutation/envelope.test.ts index a0f2b67607..66dd4cbd16 100644 --- a/scripts/mutation/envelope.test.ts +++ b/scripts/mutation/envelope.test.ts @@ -13,6 +13,39 @@ import { laneEnvelope, LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope const repoRoot = path.resolve(import.meta.dirname, '../..'); +test('invalid scope pilots and missing indexes fail before producing a measurement', () => { + const envelopePath = path.join(repoRoot, '.tmp/mutation/lane-envelope.json'); + for (const args of [ + ['--report', '.tmp/mutation/old-report.json', '--seed', '123'], + ['--scope-probe', '--modules', 'snapshot-occlusion', '--seed', '123'], + ['--scope-probe', '--list-affected'], + [ + '--scope-index', + '.tmp/mutation/missing-index.json', + '--modules', + 'snapshot-occlusion', + '--seed', + '123', + '--scope-epoch', + 'test', + ], + ]) { + const result = runCmdSync('node', ['scripts/mutation/run.ts', ...args], { + cwd: repoRoot, + allowFailure: true, + }); + assert.notEqual(result.exitCode, 0); + if (args.includes('--list-affected')) continue; + const envelope = JSON.parse(fs.readFileSync(envelopePath, 'utf8')); + assert.equal(envelope.result, 'fail'); + assert.equal( + envelope.data.modules.every((module: { score: unknown }) => module.score === null), + true, + ); + assert.match(envelope.data.error, /Scope pilots|missing-index|fresh execution/); + } +}); + function workflow(name: string): string { return fs.readFileSync(path.join(repoRoot, '.github/workflows', name), 'utf8'); } diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index 76a759bd4f..50ff1ac4da 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -18,6 +18,17 @@ import { pathToFileURL } from 'node:url'; import { runCmdStreaming, runCmdSync } from '@agent-device/host-kit/command'; import { parseScriptArgs } from '../lib/cli-args.ts'; import { laneEnvelope } from '../lib/lane-envelope.ts'; +import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.ts'; +import { + assertCoveragePolicy, + coverageScope, + scopedTestFiles, + scopeFingerprint, + type CoverageScope, + type ScopeIdentity, +} from './coverage-scope.ts'; +import { mutationSeed, MUTATION_SEED_ENV } from './seed.ts'; +import type { DryRunCompletedEvent } from '@stryker-mutator/api/report'; import { ALL_MODULE_IDS, isModuleId, @@ -46,10 +57,16 @@ const DEFAULT_REPORT_PATH = '.tmp/mutation/mutation.json'; const TEST_SCOPE_PATH = '.tmp/mutation/test-scope.json'; const ENVELOPE_PATH = '.tmp/mutation/lane-envelope.json'; const LANE_ID = 'mutation-decision-kernels'; +const COVERAGE_SCOPE_PATH = '.tmp/mutation/coverage-scope.json'; +const COVERAGE_CAPTURE_PATH = '.tmp/mutation/coverage-capture.json'; const USAGE = `Usage: pnpm mutation:run [options] --modules Restrict to specific kernel modules (default: all) + --seed Fix fast-check inputs (0..2147483647) + --scope-probe Produce fresh coverage-scope.json for one entire module + --scope-index Narrow runtime tests using a matching coverage probe + --scope-epoch Invocation identifier; required with --seed for scope pilots --affected Restrict to the modules touched between --base and HEAD --base Base ref for --affected (default: origin/main) --report Read an existing Stryker JSON report instead of running Stryker @@ -65,6 +82,28 @@ const USAGE = `Usage: pnpm mutation:run [options] --fail-envelope Write a failed lane envelope for a step that ran before (or instead of) the sweep, e.g. a self-test failure + +Local coverage pilot (opt-in; weekly defaults remain broad): + pnpm mutation:run --modules snapshot-occlusion --seed 12345 --scope-epoch run-1 --scope-probe + pnpm mutation:run --modules snapshot-occlusion --seed 12345 --scope-epoch run-1 --scope-index .tmp/mutation/coverage-scope.json + +Use a new epoch for each experiment and unchanged inputs in the same checkout/environment. +A whole-module probe can serve multiple sequential --shard consumers; save each report because +scope/report paths are shared. Consumers collect fresh coverage and execute mutants; results +are never reused. Missing, corrupt, changed or empty explicit indexes fail. + +The index fingerprints tracked/nonignored contents, inventory, permissions, tools, Node/OS +and environment (values are hashed). pnpm launcher command text is removed before test execution. +--seed fixes fast-check defaults; explicit property seeds and other randomness remain independent. +Only fresh perTest/ignoreStatic=true runs support narrowing. Static audits require separate scope. +Subprocess/thread exclusions retain the existing lane policy. Different CI job environments +are not supported. Probe success measures coverage only, never mutant outcomes. + +Before enabling CI defaults, compare fresh broad/scoped Ubuntu sweeps at one commit and seed: +identical mutant identities, no lost kills or extra uncovered mutants, explain outcome changes, +and measure both runner-minutes and critical-path time including the producer and transfers. +The first consumer can be slower; savings must amortize across consumers. Keep cross-run index +reuse and incremental results deferred. Repeat parity checks when scoping, static policy or tools change. `; type Args = { @@ -78,6 +117,10 @@ type Args = { failEnvelope: string | undefined; shard: Shard | undefined; expectShards: number | undefined; + seed: number | undefined; + scopeProbe: boolean; + scopeIndex: string | undefined; + scopeEpoch: string | undefined; }; function parseShard(value: string | undefined): Shard | undefined { @@ -119,8 +162,12 @@ function parseMutationArgs(argv: readonly string[]): Args { 'fail-envelope': { type: 'string' }, shard: { type: 'string' }, 'expect-shards': { type: 'string' }, + seed: { type: 'string' }, + 'scope-probe': { type: 'boolean', default: false }, + 'scope-index': { type: 'string' }, + 'scope-epoch': { type: 'string' }, }); - return { + const args: Args = { modules: parseModules(values.modules), affected: Boolean(values.affected), base: values.base ?? 'origin/main', @@ -131,7 +178,40 @@ function parseMutationArgs(argv: readonly string[]): Args { failEnvelope: values['fail-envelope'], shard: parseShard(values.shard), expectShards: values['expect-shards'] ? Number(values['expect-shards']) : undefined, + seed: mutationSeed(values.seed), + scopeProbe: Boolean(values['scope-probe']), + scopeIndex: values['scope-index'], + scopeEpoch: values['scope-epoch'], }; + validateMutationArgs(args); + return args; +} + +function validateMutationArgs(args: Args): void { + if (args.scopeProbe || args.scopeIndex !== undefined) { + const incompatible = [ + args.affected, + args.report, + args.reportDir, + args.listAffected, + args.failEnvelope, + args.expectShards !== undefined, + ]; + if ( + args.modules.length !== 1 || + args.seed === undefined || + !args.scopeEpoch || + (!args.scopeIndex && !args.scopeProbe) || + incompatible.some(Boolean) || + (args.scopeProbe && (args.scopeIndex !== undefined || args.shard)) + ) + throw new Error( + 'Scope pilots require one module, --seed and --scope-epoch; probe cannot be sharded or consume an index, and pilots cannot read reports or select affected modules', + ); + } else if (args.scopeEpoch !== undefined) + throw new Error('--scope-epoch requires --scope-probe or --scope-index'); + if (args.seed !== undefined && (args.report || args.reportDir)) + throw new Error('--seed requires fresh execution, not an existing report'); } /** Short, stable content hash of the Stryker config — half of a run's provenance. */ @@ -158,45 +238,132 @@ function changedFiles(base: string): string[] { return result.stdout.split('\n').filter(Boolean); } +async function mutationTestScope( + modules: readonly ModuleId[], + sources: readonly string[], + options: Args, +) { + const pilot = options.scopeProbe || Boolean(options.scopeIndex); + if (pilot) + assertCoveragePolicy(JSON.parse(fs.readFileSync(path.join(repoRoot, CONFIG_PATH), 'utf8'))); + const index: CoverageScope | undefined = options.scopeIndex + ? JSON.parse(fs.readFileSync(path.resolve(repoRoot, options.scopeIndex), 'utf8')) + : undefined; + const candidates = await relatedTestFiles(sources, repoRoot); + const identity = pilot + ? { + module: modules[0]!, + epoch: options.scopeEpoch!, + seed: options.seed!, + fingerprint: scopeFingerprint(repoRoot), + } + : undefined; + const testFiles = options.scopeIndex + ? scopedTestFiles(index!, identity!, candidates) + : candidates; + if (!testFiles.length) throw new Error(`No related mutation tests for ${modules.join(', ')}`); + return { candidates, identity, index, testFiles }; +} + +async function mutationSourceScope(modules: readonly ModuleId[], shard: Shard | undefined) { + const globs = mutateGlobs(modules); + const all = expandMutateFiles(globs, repoRoot); + if (!shard) return { mutate: globs, sources: all, sourceCount: all.length }; + const config = JSON.parse(fs.readFileSync(path.join(repoRoot, CONFIG_PATH), 'utf8')); + const mutate = await shardFiles(all, shard, repoRoot, config.mutator?.excludedMutations ?? []); + return { + mutate, + sources: [...new Set(mutate.map((file) => file.split(':')[0]!))], + sourceCount: mutate.length, + }; +} + +function finishScopeProbe( + exitCode: number, + capturePath: string, + identity: ScopeIdentity, + candidates: readonly string[], + startedAt: number, +): ScopeMeasurement { + if (exitCode !== 0 || !fs.existsSync(capturePath)) + throw new Error('Stryker scope probe failed or produced no coverage capture'); + if (scopeFingerprint(repoRoot) !== identity.fingerprint) + throw new Error('Execution inputs changed during the scope probe'); + const scope = coverageScope( + JSON.parse(fs.readFileSync(capturePath, 'utf8')) as DryRunCompletedEvent, + identity, + candidates, + repoRoot, + Date.now() - startedAt, + ); + fs.writeFileSync(path.join(repoRoot, COVERAGE_SCOPE_PATH), `${JSON.stringify(scope, null, 2)}\n`); + return { + mode: 'probe', + candidates: candidates.length, + selected: scope.covered.length, + producerMs: scope.producerMs, + }; +} + async function runStryker( modules: readonly ModuleId[], reportPath: string, shard: Shard | undefined, -): Promise { + options: Args, +): Promise { + const startedAt = Date.now(); + // pnpm appends flags to this metadata; tests must see the same environment in + // probes, broad references and scoped consumers. + delete process.env.npm_lifecycle_script; const absolute = path.isAbsolute(reportPath) ? reportPath : path.join(repoRoot, reportPath); fs.mkdirSync(path.dirname(absolute), { recursive: true }); fs.rmSync(absolute, { force: true }); + const capturePath = path.join(repoRoot, COVERAGE_CAPTURE_PATH); + if (options.scopeProbe) { + fs.rmSync(capturePath, { force: true }); + fs.rmSync(path.join(repoRoot, COVERAGE_SCOPE_PATH), { force: true }); + } // The suite Stryker replays per mutant is derived from Vitest's module graph // over the mutated files, not hand-listed; see scripts/mutation/test-scope.ts. - const globs = mutateGlobs(modules); - const all = expandMutateFiles(globs, repoRoot); - // Source ranges keep single-file kernels inside the same shard budget. - const excludedMutations = - JSON.parse(fs.readFileSync(path.join(repoRoot, CONFIG_PATH), 'utf8')).mutator - ?.excludedMutations ?? []; - const mutate = shard ? await shardFiles(all, shard, repoRoot, excludedMutations) : globs; + const { mutate, sources, sourceCount } = await mutationSourceScope(modules, shard); const scopePath = path.join(repoRoot, TEST_SCOPE_PATH); - const testFiles = await relatedTestFiles( - shard ? [...new Set(mutate.map((file) => file.split(':')[0]!))] : all, - repoRoot, + const { candidates, identity, index, testFiles } = await mutationTestScope( + modules, + sources, + options, ); - if (testFiles.length === 0) - throw new Error(`No related mutation tests for ${modules.join(', ')}`); writeTestScope(testFiles, scopePath); process.stdout.write( `mutation: ${modules.join(', ')}${shard ? ` shard ${shard.index}/${shard.count}` : ''} -> ` + - `${shard ? mutate.length : all.length} source(s), ${testFiles.length} related test file(s)\n`, + `${sourceCount} source(s), ${testFiles.length} related test file(s)\n`, ); // Stryker's `--mutate` takes one comma-separated value, not repeated args. const args = ['exec', 'stryker', 'run', CONFIG_PATH, '--mutate', mutate.join(',')]; + if (options.scopeProbe) { + args.push( + '--dryRunOnly', + '--appendPlugins', + path.join(repoRoot, 'scripts/mutation/coverage-reporter.ts'), + '--reporters', + 'scope-coverage', + ); + } const result = await runCmdStreaming('pnpm', args, { cwd: repoRoot, allowFailure: true, - env: { ...process.env, [TEST_SCOPE_ENV]: scopePath }, + env: { + ...process.env, + [TEST_SCOPE_ENV]: scopePath, + [MUTATION_SEED_ENV]: options.seed === undefined ? undefined : String(options.seed), + [COVERAGE_CAPTURE_ENV]: options.scopeProbe ? capturePath : undefined, + }, onStdoutChunk: (chunk) => void process.stdout.write(chunk), onStderrChunk: (chunk) => void process.stderr.write(chunk), }); + if (options.scopeProbe) { + return finishScopeProbe(result.exitCode, capturePath, identity!, candidates, startedAt); + } // No score threshold is configured. Any non-zero exit is a harness failure, // even if Stryker wrote a partial report before failing. if (result.exitCode !== 0 || !fs.existsSync(absolute)) { @@ -204,6 +371,14 @@ async function runStryker( `Stryker failed (exit ${result.exitCode}) or produced no report at ${reportPath}. See output above.`, ); } + if (identity && scopeFingerprint(repoRoot) !== identity.fingerprint) + throw new Error('Execution inputs changed during the scoped mutation run'); + return { + mode: index ? 'coverage' : 'related', + candidates: candidates.length, + selected: testFiles.length, + producerMs: index?.producerMs ?? null, + }; } function emit(markdown: string, summaryPath: string | undefined): void { @@ -229,6 +404,12 @@ type Stage = 'setup' | 'select' | 'stryker' | 'report' | 'score' | 'complete'; * reported fact, whereas skipping the envelope would be silence. */ const UNKNOWN_PROVENANCE: Provenance = { strykerVersion: 'unknown', configHash: 'unknown' }; +type ScopeMeasurement = { + mode: 'probe' | 'coverage' | 'related'; + candidates: number; + selected: number; + producerMs: number | null; +}; type LaneState = { stage: Stage; @@ -244,6 +425,9 @@ type LaneState = { * to prevent. */ recoveryOnly: boolean; + seed?: number; + scope?: ScopeMeasurement; + scopeProbe?: boolean; }; /** The result of an envelope already on disk, if there is a readable one. */ @@ -304,11 +488,13 @@ function writeEnvelope(state: LaneState, startedAtMs: number): void { tool: { stryker: state.provenance.strykerVersion }, configHash: state.provenance.configHash, startedAtMs, + seed: state.seed === undefined ? undefined : String(state.seed), // The lane never judges a score, so the result states whether the lane // produced a report at all. result: state.stage === 'complete' ? 'pass' : 'fail', data: { - scope: state.affected ? 'affected' : 'full-sweep', + scope: state.scopeProbe ? 'coverage-probe' : state.affected ? 'affected' : 'full-sweep', + testScope: state.scope ?? null, stage: state.stage, error: state.error ?? null, modules: state.modules.map((id) => envelopeModule(id, state.scores)), @@ -367,8 +553,10 @@ async function produceReport( modules: readonly ModuleId[], reportPath: string | undefined, shard: Shard | undefined, + args: Args, + state: LaneState, ): Promise { - if (!reportPath) await runStryker(modules, DEFAULT_REPORT_PATH, shard); + if (!reportPath) state.scope = await runStryker(modules, DEFAULT_REPORT_PATH, shard, args); return readReport(reportPath ?? DEFAULT_REPORT_PATH); } @@ -404,7 +592,8 @@ function incompleteSweepReason( async function scoreModules(args: Args, state: LaneState): Promise { state.stage = args.reportDir || args.report ? 'report' : 'stryker'; const shards = args.reportDir ? readShardedReports(args.reportDir) : undefined; - const report = shards?.report ?? (await produceReport(state.modules, args.report, args.shard)); + const report = + shards?.report ?? (await produceReport(state.modules, args.report, args.shard, args, state)); state.stage = 'score'; state.scores = summarizeReport(report, state.modules); @@ -424,6 +613,16 @@ async function sweep(args: Args, state: LaneState): Promise { return 0; } + if (args.scopeProbe) { + state.stage = 'stryker'; + state.scope = await runStryker(state.modules, DEFAULT_REPORT_PATH, undefined, args); + emit( + `## Mutation coverage probe\n\n${state.scope.selected}/${state.scope.candidates} test files cover runtime mutants. Producer cost: ${state.scope.producerMs}ms. No mutants tested.\n`, + args.summary, + ); + state.stage = 'complete'; + return 0; + } const shards = await scoreModules(args, state); const title = args.affected @@ -431,6 +630,16 @@ async function sweep(args: Args, state: LaneState): Promise { : 'Mutation score — decision kernels'; const reason = shards ? incompleteSweepReason(state.scores, shards, args) : undefined; emit(renderReport(state.scores, state.provenance, { title, incomplete: reason }), args.summary); + if (state.scope) { + const producer = + state.scope.producerMs === null + ? '' + : ` Probe cost: ${state.scope.producerMs}ms (charged separately).`; + emit( + `\nTest scope: ${state.scope.mode}; ${state.scope.selected}/${state.scope.candidates} files.${producer}\n`, + args.summary, + ); + } if (reason) throw new Error(reason); state.stage = 'complete'; return 0; @@ -495,6 +704,8 @@ async function run(argv: readonly string[], state: LaneState): Promise { state.affected = args.affected; state.provenance = readProvenance(); state.modules = args.modules; + state.seed = args.seed; + state.scopeProbe = args.scopeProbe; if (args.failEnvelope) { // A step that ran before the sweep failed (the weekly self-test, setup): the // lane produced no measurement, and that is what the envelope must say. diff --git a/scripts/mutation/seed-setup.ts b/scripts/mutation/seed-setup.ts new file mode 100644 index 0000000000..dda3f59cb1 --- /dev/null +++ b/scripts/mutation/seed-setup.ts @@ -0,0 +1,7 @@ +import { mutationSeed, MUTATION_SEED_ENV } from './seed.ts'; + +const seed = mutationSeed(process.env[MUTATION_SEED_ENV]); +if (seed !== undefined) { + const { default: fc } = await import('fast-check'); + fc.configureGlobal({ seed }); +} diff --git a/scripts/mutation/seed.test.ts b/scripts/mutation/seed.test.ts new file mode 100644 index 0000000000..1876816619 --- /dev/null +++ b/scripts/mutation/seed.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { test } from 'node:test'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { mutationSeed, MUTATION_SEED_ENV } from './seed.ts'; + +test('mutation seeds accept the fast-check integer range and reject ambiguous input', () => { + assert.equal(mutationSeed(undefined), undefined); + assert.equal(mutationSeed('0'), 0); + assert.equal(mutationSeed('2147483647'), 2147483647); + for (const value of ['', '-1', '1.5', 'NaN', '2147483648']) + assert.throws(() => mutationSeed(value), /integer/); +}); + +test('the worker setup reproduces property samples across separate processes', () => { + const sample = (seed: string) => + runCmdSync( + 'node', + [ + '--input-type=module', + '-e', + 'import fc from "fast-check"; import "./scripts/mutation/seed-setup.ts"; console.log(JSON.stringify(fc.sample(fc.integer(), 10)));', + ], + { + cwd: path.resolve(import.meta.dirname, '../..'), + env: { ...process.env, [MUTATION_SEED_ENV]: seed }, + }, + ).stdout; + assert.equal(sample('123'), sample('123')); + assert.notEqual(sample('123'), sample('124')); +}); diff --git a/scripts/mutation/seed.ts b/scripts/mutation/seed.ts new file mode 100644 index 0000000000..3cd468395d --- /dev/null +++ b/scripts/mutation/seed.ts @@ -0,0 +1,9 @@ +export const MUTATION_SEED_ENV = 'AGENT_DEVICE_MUTATION_SEED'; + +export function mutationSeed(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const seed = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(seed) || seed > 2_147_483_647) + throw new Error('Mutation seed must be an integer between 0 and 2147483647'); + return seed; +} diff --git a/vitest.mutation.config.ts b/vitest.mutation.config.ts index ef44132872..ea62e42ed9 100644 --- a/vitest.mutation.config.ts +++ b/vitest.mutation.config.ts @@ -28,7 +28,7 @@ export default defineConfig({ ...threadHostileTestFiles(repoRoot), '**/node_modules/**', ], - setupFiles: [...SETUP_FILES], + setupFiles: [...SETUP_FILES, path.join(repoRoot, 'scripts/mutation/seed-setup.ts')], ...TEST_ISOLATION, }, }); From 9e395872219d1e574d6b9fee70d9ff2a5a400166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 16:20:51 +0200 Subject: [PATCH 2/6] chore(gates): register mutation hooks and pin reporting API --- .fallowrc.json | 11 +++++++++++ package.json | 1 + pnpm-lock.yaml | 3 +++ scripts/mutation/config.test.ts | 11 +++++++++++ 4 files changed, 26 insertions(+) diff --git a/.fallowrc.json b/.fallowrc.json index 67db9a543d..26eae3ace5 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -44,6 +44,17 @@ "website/docs/404.mdx", "website/rspress.config.ts" ], + "framework": [ + { + "name": "mutation-pilot", + "enablers": ["@stryker-mutator/core"], + "entryPointRole": "test", + "entryPoints": ["scripts/mutation/coverage-reporter.ts", "scripts/mutation/seed-setup.ts"], + "usedExports": [ + { "pattern": "scripts/mutation/coverage-reporter.ts", "exports": ["strykerPlugins"] } + ] + } + ], "ignorePatterns": [ "examples/test-app/**", "scripts/perf/**", diff --git a/package.json b/package.json index f95198cb54..03dac93ad1 100644 --- a/package.json +++ b/package.json @@ -320,6 +320,7 @@ "@nkzw/eslint-plugin": "^2.0.0", "@nkzw/oxlint-config": "^1.3.0", "@statelyai/graph": "2.4.0", + "@stryker-mutator/api": "10.0.0", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/instrumenter": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4dbfc4ed3..7f929faaaf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: '@statelyai/graph': specifier: 2.4.0 version: 2.4.0(zod@4.4.3) + '@stryker-mutator/api': + specifier: 10.0.0 + version: 10.0.0 '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) diff --git a/scripts/mutation/config.test.ts b/scripts/mutation/config.test.ts index c981f2014a..f71a1e0f86 100644 --- a/scripts/mutation/config.test.ts +++ b/scripts/mutation/config.test.ts @@ -56,3 +56,14 @@ test('shard planning pins the same instrumenter version as Stryker core', () => core.dependencies['@stryker-mutator/instrumenter'], ); }); + +test('coverage reporting pins the same public API version as Stryker core', () => { + const root = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')); + const core = JSON.parse( + fs.readFileSync(path.join(repoRoot, 'node_modules/@stryker-mutator/core/package.json'), 'utf8'), + ); + assert.equal( + root.devDependencies['@stryker-mutator/api'], + core.dependencies['@stryker-mutator/api'], + ); +}); From 65500d67a14888f94420ed8d8b82ba12546531c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 17:12:31 +0200 Subject: [PATCH 3/6] fix: preserve mutation pilot provenance and test runtime support --- docs/agents/testing.md | 2 +- scripts/mutation/coverage-scope.test.ts | 6 ++++-- scripts/mutation/envelope.test.ts | 7 +++---- scripts/mutation/run.ts | 10 +++++++--- scripts/mutation/seed.test.ts | 1 + 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index f7ac40e1de..e049a264e3 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -125,7 +125,7 @@ worker process, so a case that never returns or kills its process is reported as against the exact input. Promote a discovered case with the command the harness prints — never hand-copy an unshrunk input. -Mutation scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` +Mutation scope: `scripts/mutation/modules.ts#KERNEL_MODULES` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` (PR canary). Scores are advisory; incomplete sweeps fail. Pilot protocol: `pnpm mutation:run --help`. ## Before editing a shared module diff --git a/scripts/mutation/coverage-scope.test.ts b/scripts/mutation/coverage-scope.test.ts index cff597dafc..7e59ba5acc 100644 --- a/scripts/mutation/coverage-scope.test.ts +++ b/scripts/mutation/coverage-scope.test.ts @@ -135,10 +135,12 @@ test('fingerprint invalidates source helpers, fixtures, setup, configs, new/dele fs.unlinkSync(path.join(fixture, 'test.test.ts')); assert.notEqual(scopeFingerprint(fixture), before); const withoutEnv = scopeFingerprint(fixture); - process.env.MUTATION_SCOPE_INPUT_TEST = 'changed'; + const previous = process.env.MUTATION_SCOPE_INPUT_TEST; + process.env.MUTATION_SCOPE_INPUT_TEST = previous === 'changed' ? 'changed-again' : 'changed'; try { assert.notEqual(scopeFingerprint(fixture), withoutEnv); } finally { - delete process.env.MUTATION_SCOPE_INPUT_TEST; + if (previous === undefined) delete process.env.MUTATION_SCOPE_INPUT_TEST; + else process.env.MUTATION_SCOPE_INPUT_TEST = previous; } }); diff --git a/scripts/mutation/envelope.test.ts b/scripts/mutation/envelope.test.ts index 66dd4cbd16..e11ffb34f4 100644 --- a/scripts/mutation/envelope.test.ts +++ b/scripts/mutation/envelope.test.ts @@ -30,10 +30,8 @@ test('invalid scope pilots and missing indexes fail before producing a measureme 'test', ], ]) { - const result = runCmdSync('node', ['scripts/mutation/run.ts', ...args], { - cwd: repoRoot, - allowFailure: true, - }); + fs.rmSync(envelopePath, { force: true }); + const result = runMutation(args); assert.notEqual(result.exitCode, 0); if (args.includes('--list-affected')) continue; const envelope = JSON.parse(fs.readFileSync(envelopePath, 'utf8')); @@ -43,6 +41,7 @@ test('invalid scope pilots and missing indexes fail before producing a measureme true, ); assert.match(envelope.data.error, /Scope pilots|missing-index|fresh execution/); + if (args.includes('--scope-index')) assert.equal(envelope.data.scope, 'coverage-scoped'); } }); diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index 50ff1ac4da..861de74d36 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -427,7 +427,7 @@ type LaneState = { recoveryOnly: boolean; seed?: number; scope?: ScopeMeasurement; - scopeProbe?: boolean; + pilotScope?: 'coverage-probe' | 'coverage-scoped'; }; /** The result of an envelope already on disk, if there is a readable one. */ @@ -493,7 +493,7 @@ function writeEnvelope(state: LaneState, startedAtMs: number): void { // produced a report at all. result: state.stage === 'complete' ? 'pass' : 'fail', data: { - scope: state.scopeProbe ? 'coverage-probe' : state.affected ? 'affected' : 'full-sweep', + scope: state.pilotScope ?? (state.affected ? 'affected' : 'full-sweep'), testScope: state.scope ?? null, stage: state.stage, error: state.error ?? null, @@ -705,7 +705,11 @@ async function run(argv: readonly string[], state: LaneState): Promise { state.provenance = readProvenance(); state.modules = args.modules; state.seed = args.seed; - state.scopeProbe = args.scopeProbe; + state.pilotScope = args.scopeProbe + ? 'coverage-probe' + : args.scopeIndex + ? 'coverage-scoped' + : undefined; if (args.failEnvelope) { // A step that ran before the sweep failed (the weekly self-test, setup): the // lane produced no measurement, and that is what the envelope must say. diff --git a/scripts/mutation/seed.test.ts b/scripts/mutation/seed.test.ts index 1876816619..93128fae33 100644 --- a/scripts/mutation/seed.test.ts +++ b/scripts/mutation/seed.test.ts @@ -17,6 +17,7 @@ test('the worker setup reproduces property samples across separate processes', ( runCmdSync( 'node', [ + '--experimental-strip-types', '--input-type=module', '-e', 'import fc from "fast-check"; import "./scripts/mutation/seed-setup.ts"; console.log(JSON.stringify(fc.sample(fc.integer(), 10)));', From 4fbae190b77b147e1222e8f51e9ee96cd31b359b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 17:28:30 +0200 Subject: [PATCH 4/6] fix: load mutation coverage reporter on supported Node runtimes --- ...rage-reporter.ts => coverage-reporter.mjs} | 4 +- scripts/mutation/coverage-reporter.test.ts | 59 +++++++++++++++++++ scripts/mutation/run.ts | 4 +- 3 files changed, 63 insertions(+), 4 deletions(-) rename scripts/mutation/{coverage-reporter.ts => coverage-reporter.mjs} (84%) create mode 100644 scripts/mutation/coverage-reporter.test.ts diff --git a/scripts/mutation/coverage-reporter.ts b/scripts/mutation/coverage-reporter.mjs similarity index 84% rename from scripts/mutation/coverage-reporter.ts rename to scripts/mutation/coverage-reporter.mjs index 0e98926223..4aa9f7463e 100644 --- a/scripts/mutation/coverage-reporter.ts +++ b/scripts/mutation/coverage-reporter.mjs @@ -1,10 +1,10 @@ import fs from 'node:fs'; import { declareValuePlugin, PluginKind } from '@stryker-mutator/api/plugin'; -import type { Reporter } from '@stryker-mutator/api/report'; export const COVERAGE_CAPTURE_ENV = 'AGENT_DEVICE_MUTATION_COVERAGE_CAPTURE'; -const reporter: Reporter = { +/** @type {import('@stryker-mutator/api/report').Reporter} */ +const reporter = { onDryRunCompleted(event) { const target = process.env[COVERAGE_CAPTURE_ENV]; if (!target) throw new Error('Mutation coverage capture path is missing'); diff --git a/scripts/mutation/coverage-reporter.test.ts b/scripts/mutation/coverage-reporter.test.ts new file mode 100644 index 0000000000..bb3623823c --- /dev/null +++ b/scripts/mutation/coverage-reporter.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'node:test'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { DryRunStatus } from '@stryker-mutator/api/test-runner'; +import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts'; +import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.mjs'; + +const repoRoot = path.resolve(import.meta.dirname, '../..'); + +test('Stryker loads the coverage reporter without TypeScript flags and captures a real dry run', () => { + const root = fs.realpathSync(mkdtempForTestSync('mutation-reporter-')); + fs.mkdirSync(path.join(root, 'src')); + fs.symlinkSync(path.join(repoRoot, 'node_modules'), path.join(root, 'node_modules')); + fs.writeFileSync(path.join(root, 'package.json'), '{"type":"module"}'); + fs.writeFileSync(path.join(root, 'src/decision.js'), 'export const value = true;'); + const marker = path.join(root, 'tested'); + fs.writeFileSync( + path.join(root, 'decision.test.mjs'), + `import assert from 'node:assert/strict'; import fs from 'node:fs'; import { value } from './src/decision.js'; assert.equal(value, true); fs.writeFileSync(${JSON.stringify(marker)}, 'tested');`, + ); + const capture = path.join(root, 'coverage.json'); + const config = path.join(root, 'stryker.config.json'); + fs.writeFileSync( + config, + JSON.stringify({ + mutate: ['src/decision.js'], + plugins: [], + appendPlugins: [path.join(repoRoot, 'scripts/mutation/coverage-reporter.mjs')], + testRunner: 'command', + commandRunner: { command: 'node --test decision.test.mjs' }, + coverageAnalysis: 'off', + concurrency: 1, + dryRunOnly: true, + reporters: ['scope-coverage'], + logLevel: 'fatal', + }), + ); + const result = runCmdSync( + 'node', + [path.join(repoRoot, 'node_modules/@stryker-mutator/core/bin/stryker.js'), 'run', config], + { + cwd: root, + allowFailure: true, + env: { + ...process.env, + NODE_OPTIONS: undefined, + NODE_TEST_CONTEXT: undefined, + NODE_TEST_WORKER_ID: undefined, + [COVERAGE_CAPTURE_ENV]: capture, + }, + }, + ); + assert.equal(result.exitCode, 0, result.stderr); + assert.ok(fs.existsSync(marker), result.stdout + result.stderr); + assert.equal(fs.readFileSync(marker, 'utf8'), 'tested'); + assert.equal(JSON.parse(fs.readFileSync(capture, 'utf8')).result.status, DryRunStatus.Complete); +}); diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index 861de74d36..52ce041411 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -18,7 +18,7 @@ import { pathToFileURL } from 'node:url'; import { runCmdStreaming, runCmdSync } from '@agent-device/host-kit/command'; import { parseScriptArgs } from '../lib/cli-args.ts'; import { laneEnvelope } from '../lib/lane-envelope.ts'; -import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.ts'; +import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.mjs'; import { assertCoveragePolicy, coverageScope, @@ -344,7 +344,7 @@ async function runStryker( args.push( '--dryRunOnly', '--appendPlugins', - path.join(repoRoot, 'scripts/mutation/coverage-reporter.ts'), + path.join(repoRoot, 'scripts/mutation/coverage-reporter.mjs'), '--reporters', 'scope-coverage', ); From 5f23cfea71933b07432835a7569054c6ffe641d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 17:28:30 +0200 Subject: [PATCH 5/6] chore(gates): track JavaScript mutation reporter hook --- .fallowrc.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.fallowrc.json b/.fallowrc.json index 26eae3ace5..baf47664c8 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -49,9 +49,9 @@ "name": "mutation-pilot", "enablers": ["@stryker-mutator/core"], "entryPointRole": "test", - "entryPoints": ["scripts/mutation/coverage-reporter.ts", "scripts/mutation/seed-setup.ts"], + "entryPoints": ["scripts/mutation/coverage-reporter.mjs", "scripts/mutation/seed-setup.ts"], "usedExports": [ - { "pattern": "scripts/mutation/coverage-reporter.ts", "exports": ["strykerPlugins"] } + { "pattern": "scripts/mutation/coverage-reporter.mjs", "exports": ["strykerPlugins"] } ] } ], From 617b5f4737579ba0bc0d7e0f13cecdf9af1d09bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 17:54:51 +0200 Subject: [PATCH 6/6] test: use test-kit fixtures in mutation pilot --- scripts/mutation/coverage-reporter.test.ts | 2 +- scripts/mutation/coverage-scope.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mutation/coverage-reporter.test.ts b/scripts/mutation/coverage-reporter.test.ts index bb3623823c..dd62b058a0 100644 --- a/scripts/mutation/coverage-reporter.test.ts +++ b/scripts/mutation/coverage-reporter.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { test } from 'node:test'; import { runCmdSync } from '@agent-device/host-kit/command'; import { DryRunStatus } from '@stryker-mutator/api/test-runner'; -import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts'; +import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.mjs'; const repoRoot = path.resolve(import.meta.dirname, '../..'); diff --git a/scripts/mutation/coverage-scope.test.ts b/scripts/mutation/coverage-scope.test.ts index 7e59ba5acc..95aa1220ea 100644 --- a/scripts/mutation/coverage-scope.test.ts +++ b/scripts/mutation/coverage-scope.test.ts @@ -5,7 +5,7 @@ import { test } from 'node:test'; import { runCmdSync } from '@agent-device/host-kit/command'; import type { DryRunCompletedEvent } from '@stryker-mutator/api/report'; import { DryRunStatus, TestStatus } from '@stryker-mutator/api/test-runner'; -import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts'; +import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir'; import { assertCoveragePolicy, coverageScope,