Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .fallowrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,17 @@
"website/docs/404.mdx",
"website/rspress.config.ts"
],
"framework": [
{
"name": "mutation-pilot",
"enablers": ["@stryker-mutator/core"],
"entryPointRole": "test",
"entryPoints": ["scripts/mutation/coverage-reporter.mjs", "scripts/mutation/seed-setup.ts"],
"usedExports": [
{ "pattern": "scripts/mutation/coverage-reporter.mjs", "exports": ["strykerPlugins"] }
]
}
],
"ignorePatterns": [
"examples/test-app/**",
"scripts/perf/**",
Expand Down
4 changes: 2 additions & 2 deletions docs/agents/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ worker process, so a case that never returns or kills its process is reported as
against the exact input. Promote a discovered case with the command the harness prints — never
hand-copy an unshrunk input.

Scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` (PR canary).
Scores are advisory; incomplete sweeps fail. Coverage does not prove test strength.
Mutation scope: `scripts/mutation/modules.ts#KERNEL_MODULES` (weekly), `scripts/mutation/run.ts#LANE_TOOLING`
(PR canary). Scores are advisory; incomplete sweeps fail. Pilot protocol: `pnpm mutation:run --help`.

## Before editing a shared module

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@
"@nkzw/eslint-plugin": "^2.0.0",
"@nkzw/oxlint-config": "^1.3.0",
"@statelyai/graph": "2.4.0",
"@stryker-mutator/api": "10.0.0",
"@stryker-mutator/core": "10.0.0",
"@stryker-mutator/instrumenter": "10.0.0",
"@stryker-mutator/vitest-runner": "10.0.0",
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions scripts/mutation/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,14 @@ test('shard planning pins the same instrumenter version as Stryker core', () =>
core.dependencies['@stryker-mutator/instrumenter'],
);
});

test('coverage reporting pins the same public API version as Stryker core', () => {
const root = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
const core = JSON.parse(
fs.readFileSync(path.join(repoRoot, 'node_modules/@stryker-mutator/core/package.json'), 'utf8'),
);
assert.equal(
root.devDependencies['@stryker-mutator/api'],
core.dependencies['@stryker-mutator/api'],
);
});
15 changes: 15 additions & 0 deletions scripts/mutation/coverage-reporter.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import fs from 'node:fs';
import { declareValuePlugin, PluginKind } from '@stryker-mutator/api/plugin';

export const COVERAGE_CAPTURE_ENV = 'AGENT_DEVICE_MUTATION_COVERAGE_CAPTURE';

/** @type {import('@stryker-mutator/api/report').Reporter} */
const reporter = {
onDryRunCompleted(event) {
const target = process.env[COVERAGE_CAPTURE_ENV];
if (!target) throw new Error('Mutation coverage capture path is missing');
fs.writeFileSync(target, JSON.stringify(event));
},
};

export const strykerPlugins = [declareValuePlugin(PluginKind.Reporter, 'scope-coverage', reporter)];
59 changes: 59 additions & 0 deletions scripts/mutation/coverage-reporter.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { test } from 'node:test';
import { runCmdSync } from '@agent-device/host-kit/command';
import { DryRunStatus } from '@stryker-mutator/api/test-runner';
import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir';
import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.mjs';

const repoRoot = path.resolve(import.meta.dirname, '../..');

test('Stryker loads the coverage reporter without TypeScript flags and captures a real dry run', () => {
const root = fs.realpathSync(mkdtempForTestSync('mutation-reporter-'));
fs.mkdirSync(path.join(root, 'src'));
fs.symlinkSync(path.join(repoRoot, 'node_modules'), path.join(root, 'node_modules'));
fs.writeFileSync(path.join(root, 'package.json'), '{"type":"module"}');
fs.writeFileSync(path.join(root, 'src/decision.js'), 'export const value = true;');
const marker = path.join(root, 'tested');
fs.writeFileSync(
path.join(root, 'decision.test.mjs'),
`import assert from 'node:assert/strict'; import fs from 'node:fs'; import { value } from './src/decision.js'; assert.equal(value, true); fs.writeFileSync(${JSON.stringify(marker)}, 'tested');`,
);
const capture = path.join(root, 'coverage.json');
const config = path.join(root, 'stryker.config.json');
fs.writeFileSync(
config,
JSON.stringify({
mutate: ['src/decision.js'],
plugins: [],
appendPlugins: [path.join(repoRoot, 'scripts/mutation/coverage-reporter.mjs')],
testRunner: 'command',
commandRunner: { command: 'node --test decision.test.mjs' },
coverageAnalysis: 'off',
concurrency: 1,
dryRunOnly: true,
reporters: ['scope-coverage'],
logLevel: 'fatal',
}),
);
const result = runCmdSync(
'node',
[path.join(repoRoot, 'node_modules/@stryker-mutator/core/bin/stryker.js'), 'run', config],
{
cwd: root,
allowFailure: true,
env: {
...process.env,
NODE_OPTIONS: undefined,
NODE_TEST_CONTEXT: undefined,
NODE_TEST_WORKER_ID: undefined,
[COVERAGE_CAPTURE_ENV]: capture,
},
},
);
assert.equal(result.exitCode, 0, result.stderr);
assert.ok(fs.existsSync(marker), result.stdout + result.stderr);
assert.equal(fs.readFileSync(marker, 'utf8'), 'tested');
assert.equal(JSON.parse(fs.readFileSync(capture, 'utf8')).result.status, DryRunStatus.Complete);
});
146 changes: 146 additions & 0 deletions scripts/mutation/coverage-scope.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { test } from 'node:test';
import { runCmdSync } from '@agent-device/host-kit/command';
import type { DryRunCompletedEvent } from '@stryker-mutator/api/report';
import { DryRunStatus, TestStatus } from '@stryker-mutator/api/test-runner';
import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir';
import {
assertCoveragePolicy,
coverageScope,
scopedTestFiles,
scopeFingerprint,
} from './coverage-scope.ts';

const root = path.resolve(import.meta.dirname, '../..');
const identity = {
module: 'snapshot-occlusion' as const,
epoch: 'invocation-1',
seed: 123,
fingerprint: 'inputs-1',
};
const candidates = ['src/a.test.ts', 'src/b.test.ts', 'src/unused.test.ts'];
function event(): DryRunCompletedEvent {
return {
result: {
status: DryRunStatus.Complete,
tests: candidates.map((file, index) => ({
id: String(index),
name: file,
fileName: path.join(root, file),
status: TestStatus.Success,
timeSpentMs: 1,
})),
mutantCoverage: {
static: { declaration: 1 },
perTest: { '0': { alpha: 1 }, '1': { beta: 1 }, '2': { alpha: 0 } },
},
},
timing: { net: 9, overhead: 1 },
capabilities: { reloadEnvironment: true },
};
}

test('coverage union keeps indirect covering tests, omits zero hits and intersects shard candidates', () => {
const scope = coverageScope(event(), identity, candidates, root, 20);
assert.deepEqual(scope.covered, ['src/a.test.ts', 'src/b.test.ts']);
assert.equal(scope.initialRunMs, 10);
assert.equal(scope.producerMs, 20);
assert.deepEqual(scopedTestFiles(scope, identity, ['src/b.test.ts', 'src/unused.test.ts']), [
'src/b.test.ts',
]);
assert.throws(
() => scopedTestFiles(scope, identity, ['src/unused.test.ts']),
/selected no tests/,
);
});

test('missing coverage, failed tests and unattributed covering tests cannot produce an index', () => {
const missing = event();
delete missing.result.mutantCoverage;
assert.throws(() => coverageScope(missing, identity, candidates, root, 20), /per-test/);
const failed = event();
failed.result.tests[0] = {
...failed.result.tests[0]!,
status: TestStatus.Failed,
failureMessage: 'broken',
};
assert.throws(() => coverageScope(failed, identity, candidates, root, 20), /failed tests/);
const unlocated = event();
delete unlocated.result.tests[0]!.fileName;
assert.throws(() => coverageScope(unlocated, identity, candidates, root, 20), /cannot attribute/);
const outside = event();
outside.result.tests[0]!.fileName = path.join(root, 'src/other.test.ts');
assert.throws(() => coverageScope(outside, identity, candidates, root, 20), /unexpected test/);
});

test('identity drift, new candidates and a truncated or corrupted index are rejected', () => {
const scope = coverageScope(event(), identity, candidates, root, 20);
for (const expected of [
{ ...identity, seed: 124 },
{ ...identity, epoch: 'other' },
{ ...identity, fingerprint: 'changed' },
{ ...identity, module: 'selectors' as const },
])
assert.throws(() => scopedTestFiles(scope, expected, candidates), /does not match/);
assert.throws(
() => scopedTestFiles(scope, identity, [...candidates, 'src/new.test.ts']),
/omits/,
);
assert.throws(
() => scopedTestFiles({ ...scope, covered: ['src/a.test.ts'] }, identity, candidates),
/checksum/,
);
assert.throws(
() => scopedTestFiles({ ...scope, covered: [] }, identity, candidates),
/invalid test/,
);
});

test('pilot policy rejects static audits and incremental reuse', () => {
assertCoveragePolicy({ coverageAnalysis: 'perTest', ignoreStatic: true });
for (const config of [
{ coverageAnalysis: 'all', ignoreStatic: true },
{ coverageAnalysis: 'perTest', ignoreStatic: false },
{ coverageAnalysis: 'perTest', ignoreStatic: true, incremental: true },
])
assert.throws(() => assertCoveragePolicy(config), /fresh perTest/);
});

test('fingerprint invalidates source helpers, fixtures, setup, configs, new/deleted tests and environment', () => {
const fixture = fs.realpathSync(mkdtempForTestSync('mutation-inputs-'));
runCmdSync('git', ['init', '-q', fixture]);
fs.symlinkSync(path.join(root, 'node_modules'), path.join(fixture, 'node_modules'));
fs.writeFileSync(path.join(fixture, '.gitignore'), 'node_modules\n');
for (const file of [
'helper.ts',
'fixture.json',
'setup.ts',
'stryker.config.json',
'pnpm-lock.yaml',
'test.test.ts',
]) {
const target = path.join(fixture, file);
fs.writeFileSync(target, 'before');
const before = scopeFingerprint(fixture);
fs.writeFileSync(target, 'after');
assert.notEqual(scopeFingerprint(fixture), before, file);
}
const before = scopeFingerprint(fixture);
fs.writeFileSync(path.join(fixture, 'new.test.ts'), 'new');
assert.notEqual(scopeFingerprint(fixture), before);
fs.unlinkSync(path.join(fixture, 'new.test.ts'));
assert.equal(scopeFingerprint(fixture), before);
fs.unlinkSync(path.join(fixture, 'test.test.ts'));
assert.notEqual(scopeFingerprint(fixture), before);
const withoutEnv = scopeFingerprint(fixture);
const previous = process.env.MUTATION_SCOPE_INPUT_TEST;
process.env.MUTATION_SCOPE_INPUT_TEST = previous === 'changed' ? 'changed-again' : 'changed';
try {
assert.notEqual(scopeFingerprint(fixture), withoutEnv);
} finally {
if (previous === undefined) delete process.env.MUTATION_SCOPE_INPUT_TEST;
else process.env.MUTATION_SCOPE_INPUT_TEST = previous;
}
});
Loading
Loading