Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions .github/workflows/mutation-affected.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Mutation Affected
name: Mutation Harness Canary

# PR-side half of the decision-kernel mutation lane (issue #1415).
#
Expand All @@ -9,7 +9,7 @@ name: Mutation Affected
# spends mutants on one thing only — proving the harness still runs end to end
# when the harness itself changes. `select` therefore returns an empty matrix
# unless the diff touches the lane's own sources. Selecting on derived kernel
# ownership instead would run the full ten-shard sweep on 24 of the last 40
# ownership instead would run the full sweep on 24 of the last 40
# merged PRs.
#
# On a harness diff the matrix is the canary (`LANE_CANARY`, selected by
Expand All @@ -22,14 +22,20 @@ name: Mutation Affected
on:
pull_request:
paths:
# Exactly `LANE_TOOLING` in scripts/mutation/run.ts plus this file — the
# Exactly `LANE_TOOLING` in scripts/mutation/run.ts — the
# only diffs that can produce a non-empty matrix, asserted both ways by
# scripts/mutation/workflow.test.ts. A kernel or test path here would only
# start a job that selects nothing.
- 'scripts/mutation/**'
- 'scripts/lib/**'
- 'stryker.config.json'
- 'vitest.mutation.config.ts'
- 'vitest.config.ts'
- '.github/workflows/mutation-weekly.yml'
- '.github/workflows/mutation-affected.yml'
- 'package.json'
- 'pnpm-lock.yaml'
- 'packages/*/package.json'

permissions:
contents: read
Expand All @@ -40,7 +46,7 @@ concurrency:

jobs:
select:
name: Select affected kernels
name: Select harness validation kernels
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
Expand Down Expand Up @@ -71,13 +77,13 @@ jobs:
# The self-test and the derivation run before any mutant, so a failure here
# would leave the lane with no envelope at all (#1430).
- name: Record a failed lane envelope
if: failure()
if: failure() || cancelled()
run: |
pnpm gate mutation --affected --fail-envelope \
"affected selection failed before any mutant ran (run ${{ github.run_id }})" || true

- name: Upload selection envelope
if: failure()
if: failure() || cancelled()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: mutation-affected-select
Expand All @@ -97,12 +103,15 @@ jobs:
include: ${{ fromJSON(needs.select.outputs.modules) }}
steps:
- name: Checkout
timeout-minutes: 1
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup toolchain
timeout-minutes: 2
uses: ./.github/actions/setup-node-pnpm

- name: Run mutants for ${{ matrix.name }}
timeout-minutes: 22
uses: ./.github/actions/run-gate
with:
gate: mutation
Expand All @@ -113,7 +122,7 @@ jobs:
${{ matrix.shard || '' }}

- name: Record a failed shard envelope
if: failure()
if: failure() || cancelled()
run: |
pnpm gate mutation --affected --modules ${{ matrix.module }} --fail-envelope \
"shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true
Expand Down Expand Up @@ -167,7 +176,7 @@ jobs:
fi

- name: Record a failed lane envelope
if: failure()
if: failure() || cancelled()
run: |
pnpm gate mutation --affected --fail-envelope \
"affected report failed before producing a score table (run ${{ github.run_id }})" || true
Expand Down
37 changes: 22 additions & 15 deletions .github/workflows/mutation-weekly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,8 @@ name: Mutation Weekly
# decorative"; a full-suite sweep is unaffordable, so the scope is the kernel
# registry in scripts/mutation/modules.ts and nothing else.
#
# Sharded one job per module: the whole sweep is ~2,150 mutants, and the selector
# module alone is ~1,280 of them, so a single job would sit near the 30-minute
# acceptance budget on an ubuntu runner. The shards' JSON reports are merged into
# one score table by the report job (`--report-dir`).
# Sharded from the decision-kernel registry. Oversized sources are partitioned
# at syntax boundaries; the report job merges all required shards into one table.
#
# The lane reports and never gates (#1457): the per-kernel table lands in the job
# summary and the artifact, and a low score is an input for a human-authored
Expand All @@ -31,9 +29,7 @@ jobs:
shard:
name: Mutants (${{ matrix.name }})
runs-on: ubuntu-latest
# The acceptance budget is 30 minutes of wall clock for the lane, and shards
# run in parallel, so the budget is per shard. The observed rate on a 2-core
# runner is ~3s/mutant, which is why selectors (~1,280 mutants) is sliced.
# Bound setup and mutation separately, leaving headroom for evidence and upload.
timeout-minutes: 30
strategy:
fail-fast: false
Expand All @@ -42,24 +38,35 @@ jobs:
# scripts/mutation/workflow.test.ts, which also pins --expect-shards to
# the number of entries here.
include:
- { name: kernel-errors, module: kernel-errors }
- { name: kernel-errors-1, module: kernel-errors, shard: 1/5 }
- { name: kernel-errors-2, module: kernel-errors, shard: 2/5 }
- { name: kernel-errors-3, module: kernel-errors, shard: 3/5 }
- { name: kernel-errors-4, module: kernel-errors, shard: 4/5 }
- { name: kernel-errors-5, module: kernel-errors, shard: 5/5 }
- { name: daemon-ref-frame, module: daemon-ref-frame }
- { name: interaction-settle, module: interaction-settle }
- { name: scroll-edge-state, module: scroll-edge-state }
- { name: selectors-1, module: selectors, shard: 1/4 }
- { name: selectors-2, module: selectors, shard: 2/4 }
- { name: selectors-3, module: selectors, shard: 3/4 }
- { name: selectors-4, module: selectors, shard: 4/4 }
- { name: selectors-1, module: selectors, shard: 1/8 }
- { name: selectors-2, module: selectors, shard: 2/8 }
- { name: selectors-3, module: selectors, shard: 3/8 }
- { name: selectors-4, module: selectors, shard: 4/8 }
- { name: selectors-5, module: selectors, shard: 5/8 }
- { name: selectors-6, module: selectors, shard: 6/8 }
- { name: selectors-7, module: selectors, shard: 7/8 }
- { name: selectors-8, module: selectors, shard: 8/8 }
- { name: target-annotation-serde, module: target-annotation-serde }
- { name: snapshot-occlusion, module: snapshot-occlusion }
steps:
- name: Checkout
timeout-minutes: 1
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup toolchain
timeout-minutes: 2
uses: ./.github/actions/setup-node-pnpm

- name: Run mutants for ${{ matrix.name }}
timeout-minutes: 22
uses: ./.github/actions/run-gate
with:
gate: mutation
Expand All @@ -73,7 +80,7 @@ jobs:
# without this the artifact is absent and "shard failed" is indistinguishable
# from "lane never ran". --fail-envelope leaves a real verdict untouched.
- name: Record a failed shard envelope
if: failure()
if: failure() || cancelled()
run: |
pnpm gate mutation --modules ${{ matrix.module }} --fail-envelope \
"shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true
Expand Down Expand Up @@ -126,12 +133,12 @@ jobs:
--report-dir
.tmp/mutation/shards
--expect-shards
10
18

# The self-test and the artifact download both run before the scoring step,
# so a failure there would otherwise leave the aggregate lane with no envelope.
- name: Record a failed lane envelope
if: failure()
if: failure() || cancelled()
run: |
pnpm gate mutation --fail-envelope \
"weekly report job failed before producing a score table (run ${{ github.run_id }})" || true
Expand Down
4 changes: 2 additions & 2 deletions docs/agents/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ worker process, so a case that never returns or kills its process is reported as
against the exact input. Promote a discovered case with the command the harness prints — never
hand-copy an unshrunk input.

Mutation is report-only and limited to the registry in `scripts/mutation/modules.ts`. It measures
whether tests distinguish changed decision logic. Do not infer redundancy from line coverage alone.
Scope: `scripts/mutation/modules.ts` (weekly), `scripts/mutation/run.ts#LANE_TOOLING` (PR canary).
Scores are advisory; incomplete sweeps fail. Coverage does not prove test strength.

## Before editing a shared module

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@
"@nkzw/oxlint-config": "^1.3.0",
"@statelyai/graph": "2.4.0",
"@stryker-mutator/core": "10.0.0",
"@stryker-mutator/instrumenter": "10.0.0",
"@stryker-mutator/vitest-runner": "10.0.0",
"@types/json-schema": "^7.0.15",
"@types/node": "^22.20.1",
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions scripts/mutation/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,20 @@ test('the config content hash is stable and content-addressed', () => {
assert.notEqual(configHash('a'), configHash('b'));
assert.match(configHash('a'), /^sha256:[0-9a-f]{12}$/);
});

test('stryker disables per-mutant related discovery and uses perTest coverage', () => {
const config = readConfig();
assert.equal((config.vitest as { related: boolean }).related, false);
assert.equal(config.coverageAnalysis, 'perTest');
});

test('shard planning pins the same instrumenter version as Stryker core', () => {
const root = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
const core = JSON.parse(
fs.readFileSync(path.join(repoRoot, 'node_modules/@stryker-mutator/core/package.json'), 'utf8'),
);
assert.equal(
root.devDependencies['@stryker-mutator/instrumenter'],
core.dependencies['@stryker-mutator/instrumenter'],
);
});
Loading
Loading