Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions .github/actions/test-mutations/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ inputs:
publish:
description: Publish only audited ordinary main-push results
required: true
parallel-fallback:
description: Hand a cold hosted PR to the full partition workflow
default: 'false'
outputs:
route:
description: Selected native execution or full partition handoff
value: ${{ steps.mutation.outputs.route }}
runs:
using: composite
steps:
Expand Down Expand Up @@ -58,7 +65,7 @@ runs:
core.setOutput('run', run.id);
core.setOutput('commit', run.head_sha);
} catch {
core.notice('Mutation baseline unavailable; native full execution will run.');
core.notice('Mutation baseline unavailable; fresh full execution is required.');
}
- name: Download the selected immutable file with native digest verification
id: download
Expand Down Expand Up @@ -88,29 +95,46 @@ runs:
path: .cache/mutation/pr-baseline.json
key: mutation-pr-v1-${{ github.repository_id }}-${{ github.event.pull_request.number }}-${{ steps.pr-key.outputs.key }}
- name: Run ordinary tests, full discovery and the adjusted mutation audit
id: mutation
shell: bash
env:
FORCE_FULL: ${{ inputs.full }}
PARALLEL_FALLBACK: ${{ inputs.parallel-fallback }}
MUTATION_PR_CACHE_RESTORED: ${{ steps.pr-cache.outcome == 'success' && steps.pr-cache.outputs.cache-hit == 'true' }}
MUTATION_BASELINE_SHA: ${{ steps.download.outcome == 'success' && steps.baseline.outputs.commit || '' }}
run: |
if [ "$FORCE_FULL" = true ]; then
node scripts/run-mutations.mjs --full
echo 'route=native' >> "$GITHUB_OUTPUT"
elif [ "$PARALLEL_FALLBACK" = true ]; then
node scripts/run-mutations.mjs --parallel-fallback
else
node scripts/run-mutations.mjs
echo 'route=native' >> "$GITHUB_OUTPUT"
fi
- name: Require an explicit ordinary execution route
if: inputs.parallel-fallback == 'true' && inputs.full != 'true'
shell: bash
env:
ROUTE: ${{ steps.mutation.outputs.route }}
run: |
node --input-type=module <<'JS'
import assert from 'node:assert/strict';
assert.ok(['native', 'full-sharded'].includes(process.env.ROUTE), 'Missing or invalid mutation route');
JS
- name: Require active genuine encoder mutation coverage
if: inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native'
shell: bash
run: node scripts/verify-encoder-discovery.mjs reports/mutation/mutation.json
- name: Save own-PR results only after mutation and encoder checks
if: success() && steps.pr-key.outputs.key != '' && steps.pr-cache.outputs.cache-hit != 'true' && hashFiles('.cache/mutation/pr-baseline.json') != ''
if: success() && (inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native') && steps.pr-key.outputs.key != '' && steps.pr-cache.outputs.cache-hit != 'true' && hashFiles('.cache/mutation/pr-baseline.json') != ''
continue-on-error: true
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: .cache/mutation/pr-baseline.json
key: mutation-pr-v1-${{ github.repository_id }}-${{ github.event.pull_request.number }}-${{ steps.pr-key.outputs.key }}
- name: Publish audited main results; consumers require the whole CI run to succeed
if: success() && inputs.publish == 'true' && hashFiles('.cache/mutation/native-baseline.json') != ''
if: success() && (inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native') && inputs.publish == 'true' && hashFiles('.cache/mutation/native-baseline.json') != ''
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
path: .cache/mutation/native-baseline.json
Expand Down
11 changes: 8 additions & 3 deletions .github/workflows/mutation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ jobs:
if: ${{ !inputs.full }}
runs-on: ubuntu-24.04
timeout-minutes: 100
outputs:
route: ${{ steps.mutation.outputs.route }}
steps:
# Supported callers bind ref to their event SHA; reject drift before executing source.
- name: Require the caller's exact commit
Expand All @@ -53,14 +55,17 @@ jobs:
persist-credentials: false
- uses: ./.github/actions/setup-project
- uses: ./.github/actions/test-mutations
id: mutation
with:
base: ${{ inputs.base }}
full: 'false'
publish: ${{ inputs.publish }}
parallel-fallback: ${{ github.event_name == 'pull_request' }}

plan:
name: Discover full parser partitions
if: inputs.full
if: ${{ !cancelled() && (inputs.full || (needs.ordinary.result == 'success' && needs.ordinary.outputs.route == 'full-sharded')) }}
needs: ordinary
runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
Expand All @@ -87,7 +92,7 @@ jobs:

shard:
name: Fresh parser partition ${{ matrix.shard }}
if: inputs.full
if: ${{ !cancelled() && needs.plan.result == 'success' }}
needs: plan
strategy:
fail-fast: false
Expand Down Expand Up @@ -117,7 +122,7 @@ jobs:

collect:
name: Audit complete fresh parser evidence
if: always() && inputs.full
if: always() && needs.plan.result != 'skipped'
needs: [plan, shard]
runs-on: ubuntu-24.04
timeout-minutes: 15
Expand Down
34 changes: 30 additions & 4 deletions docs/mutation-cache.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ A compatible completed baseline is supplied unchanged through Stryker's native
narrow `--mutate` on this ordinary path or preserve an old inventory in place of
current discovery. `npm run mutation:full` remains a single fresh local campaign.

Scheduled CI, full manual CI and exact-source release validation instead use the
native GitHub matrix in `.github/workflows/mutation.yml`. Node's filesystem glob
Cold ordinary PRs, scheduled CI, full manual CI and exact-source release validation
use the native GitHub matrix in `.github/workflows/mutation.yml`. Node's filesystem glob
discovers every parser source. The pinned Stryker instrumenter generates the
complete current mutant inventory without running tests. A small planner joins
overlapping mutant spans into indivisible components and assigns larger components
Expand Down Expand Up @@ -57,7 +57,17 @@ then uses Vitest's runnable-test discovery for report file coverage. This permit
native all-skipped suites (such as an absent optional external sample) without a
filename exception; every runnable suite must appear consistently in every shard.

Ordinary PR/main runs, including cache misses, retain the single-runner path.
Ordinary hosted PR runs invoke `--parallel-fallback`, which performs the same
baseline admission as the ordinary runner. A PR with a compatible baseline emits
`route=native` through `GITHUB_OUTPUT` and continues on the single-runner path.
A cold PR emits `route=full-sharded` and returns before test discovery, the ordinary
suite or Stryker execution; the workflow then requires the full matrix and its
collector. Main pushes keep the ordinary native command; the action reports
`route=native`, including cache misses, so a successful cold main run can publish
the native baseline needed by later runs.
The route changes scheduling only: input classification, complete scope, fresh
coverage, source-bound proofs and the 100% adjusted gate remain the same.

Mutant counts are only a scheduling heuristic: individual tests and timeouts vary
widely, and a method-spanning mutant prevents further safe subdivision. The
planner uses no historical timing store, result cache, dependency graph or report
Expand Down Expand Up @@ -146,12 +156,20 @@ accepted. The entire producing CI run must be completed successfully.

Pinned official `actions/download-artifact` downloads the selected immutable
artifact ID from that repository/run with digest mismatch set to `error`.
Transport failure falls back to native full execution. The envelope's commit and
Transport failure is a cache miss: ordinary PRs use the full matrix, while main
and local invocations execute the full native campaign. The envelope's commit and
key must match, and its native report must pass the independent score audit.
Only ordinary main pushes upload an audited baseline; official
`actions/upload-artifact` publishes one unzipped JSON file. No custom archive
parser or artifact authentication client is maintained.

Admission diagnostics distinguish `native-baseline-missing`,
`native-baseline-incompatible`, `native-baseline-corrupt-or-unauditable`, and
`native-baseline-unreadable`. All four mean no admitted report and require fresh
full execution through the applicable route; none bypasses the gate. A compatible
report is logged as `native-same-input-reuse`. Explicit full mode, an unavailable
input fingerprint, or a supplied `NODE_OPTIONS` is logged as `native-full-required`.

### Optional own-PR cache (lower trust)

Normal `pull_request` merge-ref runs may also use official `actions/cache` for
Expand All @@ -170,6 +188,12 @@ mutation and encoder checks, not after all sibling CI jobs. Main and release
runs never read this file; forced-full mode bypasses both result sources.
`pull_request_target`, dispatches and non-merge refs cannot save or consume it.

Cold PR matrix runs do not publish either native baseline envelope. Their raw
shard reports are not merged into an incremental report, so repeated runs of a
cold PR can remain cold until a compatible main baseline becomes available.
With hosted routing enabled, an own-PR cache can be saved after a successful native
run that admitted a compatible baseline; the matrix does not seed it.

Restore/save failures, eviction, partial/corrupt files or incompatible reports
fall back to full execution. Native caches are immutable: an unusable exact-key
entry cannot be repaired in place and may remain cold until eviction or a
Expand Down Expand Up @@ -214,6 +238,8 @@ makes no whole-parser speedup claim. Survivors still require all selected tests.
`npm run test:mutation-cache:integration` is a short real Stryker/Vitest toy:
unchanged/docs reuse, forced-full parity and source/helper/fixture/config/lock
invalidation. Weakening the helper or fixture must expose an actual survivor.
It exercises native execution and admission helpers, not the hosted CLI routing
or matrix handoff; focused routing and workflow tests cover those contracts.
The existing required cache-regression check name is retained, but the former
large custom-framework campaign is removed with that framework.

Expand Down
27 changes: 22 additions & 5 deletions scripts/mutation-cache.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export const WORKING_REPORT = '.cache/mutation/working.json';
export const BASELINE = '.cache/mutation/native-baseline.json';
export const PR_BASELINE = '.cache/mutation/pr-baseline.json';
const sourceIdentity = /^(?:[a-f0-9]{40}|local|pr:[1-9]\d*:[1-9]\d*)$/;
class IncompatibleCacheError extends Error {}

export function pullRequestIdentity(env) {
const ref = /^refs\/pull\/([1-9]\d*)\/merge$/.exec(env.GITHUB_REF ?? '');
Expand All @@ -29,22 +30,38 @@ export function validateCache(baseline, key, identity, audit) {
baseline.key !== key ||
baseline[identity.startsWith('pr:') ? 'prIdentity' : 'sourceCommit'] !== identity
)
throw new Error('No compatible native baseline for this scope');
throw new IncompatibleCacheError('No compatible native baseline for this scope');
audit(baseline.report, { nativeReuse: true });
return baseline.report;
}

export async function readCache(root, key, identity, audit) {
let bytes;
try {
const baseline = JSON.parse(
await readFile(join(root, identity?.startsWith('pr:') ? PR_BASELINE : BASELINE), 'utf8'),
bytes = await readFile(
join(root, identity?.startsWith('pr:') ? PR_BASELINE : BASELINE),
'utf8',
);
} catch (error) {
return {
report: null,
reason: error.code === 'ENOENT' ? 'native-baseline-missing' : 'native-baseline-unreadable',
};
}
try {
const baseline = JSON.parse(bytes);
return {
report: validateCache(baseline, key, identity, audit),
reason: 'native-same-input-reuse',
};
} catch {
return { report: null, reason: 'native-baseline-missing-or-incompatible' };
} catch (error) {
return {
report: null,
reason:
error instanceof IncompatibleCacheError
? 'native-baseline-incompatible'
: 'native-baseline-corrupt-or-unauditable',
};
}
}

Expand Down
30 changes: 21 additions & 9 deletions scripts/mutation-cache.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -113,27 +113,39 @@ test('missing, malformed, mismatched, incomplete and unauditable baselines are c
assert.deepEqual(value, report);
assert.deepEqual(options, { nativeReuse: true });
};
assert.equal((await readCache(root, key, commit, audit)).report, null);
assert.deepEqual(await readCache(root, key, commit, audit), {
report: null,
reason: 'native-baseline-missing',
});
await put(BASELINE, '{');
assert.equal((await readCache(root, key, commit, audit)).report, null);
assert.deepEqual(await readCache(root, key, commit, audit), {
report: null,
reason: 'native-baseline-corrupt-or-unauditable',
});
await promoteCache(root, report, key, commit);
const baseline = JSON.parse(await readFile(join(root, BASELINE), 'utf8'));
for (const delta of [
{ completed: false },
{ key: 'c'.repeat(64) },
{ sourceCommit: 'd'.repeat(40) },
{ schemaVersion: 0 },
{ report: { survived: true } },
for (const [delta, reason] of [
[{ completed: false }, 'native-baseline-incompatible'],
[{ key: 'c'.repeat(64) }, 'native-baseline-incompatible'],
[{ sourceCommit: 'd'.repeat(40) }, 'native-baseline-incompatible'],
[{ schemaVersion: 0 }, 'native-baseline-incompatible'],
[{ report: { survived: true } }, 'native-baseline-corrupt-or-unauditable'],
]) {
await put(BASELINE, JSON.stringify({ ...baseline, ...delta }));
assert.equal((await readCache(root, key, commit, audit)).report, null);
assert.deepEqual(await readCache(root, key, commit, audit), { report: null, reason });
}
assert.throws(() => validateCache(baseline, key, null, audit));
assert.throws(() => validateCache(baseline, key, 'local', audit));
await assert.rejects(promoteCache(root, report, 'short', commit));
await promoteCache(root, report, key, 'local');
assert.deepEqual((await readCache(root, key, 'local', audit)).report, report);
assert.equal((await readCache(root, key, commit, audit)).report, null);
await rm(join(root, BASELINE));
await mkdir(join(root, BASELINE));
assert.deepEqual(await readCache(root, key, commit, audit), {
report: null,
reason: 'native-baseline-unreadable',
});
});

test('admission preserves native report bytes semantically and reports native reuse honestly', async (t) => {
Expand Down
Loading
Loading