diff --git a/.github/actions/test-mutations/action.yml b/.github/actions/test-mutations/action.yml index 97feec1..c971b96 100644 --- a/.github/actions/test-mutations/action.yml +++ b/.github/actions/test-mutations/action.yml @@ -10,6 +10,13 @@ inputs: publish: description: Publish only audited ordinary main-push results required: true + parallel-fallback: + description: Hand a cold hosted PR to the full partition workflow + default: 'false' +outputs: + route: + description: Selected native execution or full partition handoff + value: ${{ steps.mutation.outputs.route }} runs: using: composite steps: @@ -58,7 +65,7 @@ runs: core.setOutput('run', run.id); core.setOutput('commit', run.head_sha); } catch { - core.notice('Mutation baseline unavailable; native full execution will run.'); + core.notice('Mutation baseline unavailable; fresh full execution is required.'); } - name: Download the selected immutable file with native digest verification id: download @@ -88,29 +95,46 @@ runs: path: .cache/mutation/pr-baseline.json key: mutation-pr-v1-${{ github.repository_id }}-${{ github.event.pull_request.number }}-${{ steps.pr-key.outputs.key }} - name: Run ordinary tests, full discovery and the adjusted mutation audit + id: mutation shell: bash env: FORCE_FULL: ${{ inputs.full }} + PARALLEL_FALLBACK: ${{ inputs.parallel-fallback }} MUTATION_PR_CACHE_RESTORED: ${{ steps.pr-cache.outcome == 'success' && steps.pr-cache.outputs.cache-hit == 'true' }} MUTATION_BASELINE_SHA: ${{ steps.download.outcome == 'success' && steps.baseline.outputs.commit || '' }} run: | if [ "$FORCE_FULL" = true ]; then node scripts/run-mutations.mjs --full + echo 'route=native' >> "$GITHUB_OUTPUT" + elif [ "$PARALLEL_FALLBACK" = true ]; then + node scripts/run-mutations.mjs --parallel-fallback else node scripts/run-mutations.mjs + echo 'route=native' >> "$GITHUB_OUTPUT" fi + - name: Require an explicit ordinary execution route + if: inputs.parallel-fallback == 'true' && inputs.full != 'true' + shell: bash + env: + ROUTE: ${{ steps.mutation.outputs.route }} + run: | + node --input-type=module <<'JS' + import assert from 'node:assert/strict'; + assert.ok(['native', 'full-sharded'].includes(process.env.ROUTE), 'Missing or invalid mutation route'); + JS - name: Require active genuine encoder mutation coverage + if: inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native' shell: bash run: node scripts/verify-encoder-discovery.mjs reports/mutation/mutation.json - name: Save own-PR results only after mutation and encoder checks - if: success() && steps.pr-key.outputs.key != '' && steps.pr-cache.outputs.cache-hit != 'true' && hashFiles('.cache/mutation/pr-baseline.json') != '' + if: success() && (inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native') && steps.pr-key.outputs.key != '' && steps.pr-cache.outputs.cache-hit != 'true' && hashFiles('.cache/mutation/pr-baseline.json') != '' continue-on-error: true uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .cache/mutation/pr-baseline.json key: mutation-pr-v1-${{ github.repository_id }}-${{ github.event.pull_request.number }}-${{ steps.pr-key.outputs.key }} - name: Publish audited main results; consumers require the whole CI run to succeed - if: success() && inputs.publish == 'true' && hashFiles('.cache/mutation/native-baseline.json') != '' + if: success() && (inputs.parallel-fallback != 'true' || inputs.full == 'true' || steps.mutation.outputs.route == 'native') && inputs.publish == 'true' && hashFiles('.cache/mutation/native-baseline.json') != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: path: .cache/mutation/native-baseline.json diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index c41b54c..e1104d7 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -36,6 +36,8 @@ jobs: if: ${{ !inputs.full }} runs-on: ubuntu-24.04 timeout-minutes: 100 + outputs: + route: ${{ steps.mutation.outputs.route }} steps: # Supported callers bind ref to their event SHA; reject drift before executing source. - name: Require the caller's exact commit @@ -53,14 +55,17 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-project - uses: ./.github/actions/test-mutations + id: mutation with: base: ${{ inputs.base }} full: 'false' publish: ${{ inputs.publish }} + parallel-fallback: ${{ github.event_name == 'pull_request' }} plan: name: Discover full parser partitions - if: inputs.full + if: ${{ !cancelled() && (inputs.full || (needs.ordinary.result == 'success' && needs.ordinary.outputs.route == 'full-sharded')) }} + needs: ordinary runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: @@ -87,7 +92,7 @@ jobs: shard: name: Fresh parser partition ${{ matrix.shard }} - if: inputs.full + if: ${{ !cancelled() && needs.plan.result == 'success' }} needs: plan strategy: fail-fast: false @@ -117,7 +122,7 @@ jobs: collect: name: Audit complete fresh parser evidence - if: always() && inputs.full + if: always() && needs.plan.result != 'skipped' needs: [plan, shard] runs-on: ubuntu-24.04 timeout-minutes: 15 diff --git a/docs/mutation-cache.md b/docs/mutation-cache.md index 44809ad..d88eee0 100644 --- a/docs/mutation-cache.md +++ b/docs/mutation-cache.md @@ -16,8 +16,8 @@ A compatible completed baseline is supplied unchanged through Stryker's native narrow `--mutate` on this ordinary path or preserve an old inventory in place of current discovery. `npm run mutation:full` remains a single fresh local campaign. -Scheduled CI, full manual CI and exact-source release validation instead use the -native GitHub matrix in `.github/workflows/mutation.yml`. Node's filesystem glob +Cold ordinary PRs, scheduled CI, full manual CI and exact-source release validation +use the native GitHub matrix in `.github/workflows/mutation.yml`. Node's filesystem glob discovers every parser source. The pinned Stryker instrumenter generates the complete current mutant inventory without running tests. A small planner joins overlapping mutant spans into indivisible components and assigns larger components @@ -57,7 +57,17 @@ then uses Vitest's runnable-test discovery for report file coverage. This permit native all-skipped suites (such as an absent optional external sample) without a filename exception; every runnable suite must appear consistently in every shard. -Ordinary PR/main runs, including cache misses, retain the single-runner path. +Ordinary hosted PR runs invoke `--parallel-fallback`, which performs the same +baseline admission as the ordinary runner. A PR with a compatible baseline emits +`route=native` through `GITHUB_OUTPUT` and continues on the single-runner path. +A cold PR emits `route=full-sharded` and returns before test discovery, the ordinary +suite or Stryker execution; the workflow then requires the full matrix and its +collector. Main pushes keep the ordinary native command; the action reports +`route=native`, including cache misses, so a successful cold main run can publish +the native baseline needed by later runs. +The route changes scheduling only: input classification, complete scope, fresh +coverage, source-bound proofs and the 100% adjusted gate remain the same. + Mutant counts are only a scheduling heuristic: individual tests and timeouts vary widely, and a method-spanning mutant prevents further safe subdivision. The planner uses no historical timing store, result cache, dependency graph or report @@ -146,12 +156,20 @@ accepted. The entire producing CI run must be completed successfully. Pinned official `actions/download-artifact` downloads the selected immutable artifact ID from that repository/run with digest mismatch set to `error`. -Transport failure falls back to native full execution. The envelope's commit and +Transport failure is a cache miss: ordinary PRs use the full matrix, while main +and local invocations execute the full native campaign. The envelope's commit and key must match, and its native report must pass the independent score audit. Only ordinary main pushes upload an audited baseline; official `actions/upload-artifact` publishes one unzipped JSON file. No custom archive parser or artifact authentication client is maintained. +Admission diagnostics distinguish `native-baseline-missing`, +`native-baseline-incompatible`, `native-baseline-corrupt-or-unauditable`, and +`native-baseline-unreadable`. All four mean no admitted report and require fresh +full execution through the applicable route; none bypasses the gate. A compatible +report is logged as `native-same-input-reuse`. Explicit full mode, an unavailable +input fingerprint, or a supplied `NODE_OPTIONS` is logged as `native-full-required`. + ### Optional own-PR cache (lower trust) Normal `pull_request` merge-ref runs may also use official `actions/cache` for @@ -170,6 +188,12 @@ mutation and encoder checks, not after all sibling CI jobs. Main and release runs never read this file; forced-full mode bypasses both result sources. `pull_request_target`, dispatches and non-merge refs cannot save or consume it. +Cold PR matrix runs do not publish either native baseline envelope. Their raw +shard reports are not merged into an incremental report, so repeated runs of a +cold PR can remain cold until a compatible main baseline becomes available. +With hosted routing enabled, an own-PR cache can be saved after a successful native +run that admitted a compatible baseline; the matrix does not seed it. + Restore/save failures, eviction, partial/corrupt files or incompatible reports fall back to full execution. Native caches are immutable: an unusable exact-key entry cannot be repaired in place and may remain cold until eviction or a @@ -214,6 +238,8 @@ makes no whole-parser speedup claim. Survivors still require all selected tests. `npm run test:mutation-cache:integration` is a short real Stryker/Vitest toy: unchanged/docs reuse, forced-full parity and source/helper/fixture/config/lock invalidation. Weakening the helper or fixture must expose an actual survivor. +It exercises native execution and admission helpers, not the hosted CLI routing +or matrix handoff; focused routing and workflow tests cover those contracts. The existing required cache-regression check name is retained, but the former large custom-framework campaign is removed with that framework. diff --git a/scripts/mutation-cache.mjs b/scripts/mutation-cache.mjs index 12f0db7..cfdf808 100644 --- a/scripts/mutation-cache.mjs +++ b/scripts/mutation-cache.mjs @@ -6,6 +6,7 @@ export const WORKING_REPORT = '.cache/mutation/working.json'; export const BASELINE = '.cache/mutation/native-baseline.json'; export const PR_BASELINE = '.cache/mutation/pr-baseline.json'; const sourceIdentity = /^(?:[a-f0-9]{40}|local|pr:[1-9]\d*:[1-9]\d*)$/; +class IncompatibleCacheError extends Error {} export function pullRequestIdentity(env) { const ref = /^refs\/pull\/([1-9]\d*)\/merge$/.exec(env.GITHUB_REF ?? ''); @@ -29,22 +30,38 @@ export function validateCache(baseline, key, identity, audit) { baseline.key !== key || baseline[identity.startsWith('pr:') ? 'prIdentity' : 'sourceCommit'] !== identity ) - throw new Error('No compatible native baseline for this scope'); + throw new IncompatibleCacheError('No compatible native baseline for this scope'); audit(baseline.report, { nativeReuse: true }); return baseline.report; } export async function readCache(root, key, identity, audit) { + let bytes; try { - const baseline = JSON.parse( - await readFile(join(root, identity?.startsWith('pr:') ? PR_BASELINE : BASELINE), 'utf8'), + bytes = await readFile( + join(root, identity?.startsWith('pr:') ? PR_BASELINE : BASELINE), + 'utf8', ); + } catch (error) { + return { + report: null, + reason: error.code === 'ENOENT' ? 'native-baseline-missing' : 'native-baseline-unreadable', + }; + } + try { + const baseline = JSON.parse(bytes); return { report: validateCache(baseline, key, identity, audit), reason: 'native-same-input-reuse', }; - } catch { - return { report: null, reason: 'native-baseline-missing-or-incompatible' }; + } catch (error) { + return { + report: null, + reason: + error instanceof IncompatibleCacheError + ? 'native-baseline-incompatible' + : 'native-baseline-corrupt-or-unauditable', + }; } } diff --git a/scripts/mutation-cache.test.mjs b/scripts/mutation-cache.test.mjs index 5359834..490f703 100644 --- a/scripts/mutation-cache.test.mjs +++ b/scripts/mutation-cache.test.mjs @@ -113,20 +113,26 @@ test('missing, malformed, mismatched, incomplete and unauditable baselines are c assert.deepEqual(value, report); assert.deepEqual(options, { nativeReuse: true }); }; - assert.equal((await readCache(root, key, commit, audit)).report, null); + assert.deepEqual(await readCache(root, key, commit, audit), { + report: null, + reason: 'native-baseline-missing', + }); await put(BASELINE, '{'); - assert.equal((await readCache(root, key, commit, audit)).report, null); + assert.deepEqual(await readCache(root, key, commit, audit), { + report: null, + reason: 'native-baseline-corrupt-or-unauditable', + }); await promoteCache(root, report, key, commit); const baseline = JSON.parse(await readFile(join(root, BASELINE), 'utf8')); - for (const delta of [ - { completed: false }, - { key: 'c'.repeat(64) }, - { sourceCommit: 'd'.repeat(40) }, - { schemaVersion: 0 }, - { report: { survived: true } }, + for (const [delta, reason] of [ + [{ completed: false }, 'native-baseline-incompatible'], + [{ key: 'c'.repeat(64) }, 'native-baseline-incompatible'], + [{ sourceCommit: 'd'.repeat(40) }, 'native-baseline-incompatible'], + [{ schemaVersion: 0 }, 'native-baseline-incompatible'], + [{ report: { survived: true } }, 'native-baseline-corrupt-or-unauditable'], ]) { await put(BASELINE, JSON.stringify({ ...baseline, ...delta })); - assert.equal((await readCache(root, key, commit, audit)).report, null); + assert.deepEqual(await readCache(root, key, commit, audit), { report: null, reason }); } assert.throws(() => validateCache(baseline, key, null, audit)); assert.throws(() => validateCache(baseline, key, 'local', audit)); @@ -134,6 +140,12 @@ test('missing, malformed, mismatched, incomplete and unauditable baselines are c await promoteCache(root, report, key, 'local'); assert.deepEqual((await readCache(root, key, 'local', audit)).report, report); assert.equal((await readCache(root, key, commit, audit)).report, null); + await rm(join(root, BASELINE)); + await mkdir(join(root, BASELINE)); + assert.deepEqual(await readCache(root, key, commit, audit), { + report: null, + reason: 'native-baseline-unreadable', + }); }); test('admission preserves native report bytes semantically and reports native reuse honestly', async (t) => { diff --git a/scripts/mutation-routing-workflow.test.mjs b/scripts/mutation-routing-workflow.test.mjs new file mode 100644 index 0000000..0998f86 --- /dev/null +++ b/scripts/mutation-routing-workflow.test.mjs @@ -0,0 +1,237 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { parse } from 'yaml'; + +const read = async (file) => parse(await readFile(new URL(file, import.meta.url), 'utf8')); +const action = await read('../.github/actions/test-mutations/action.yml'); +const { jobs } = await read('../.github/workflows/mutation.yml'); +const steps = action.runs.steps; +const execution = steps.find((step) => step.id === 'mutation'); +const routeGuard = steps.find( + (step) => step.name === 'Require an explicit ordinary execution route', +); +const encoder = steps.find((step) => + step.run?.startsWith('node scripts/verify-encoder-discovery.mjs'), +); +const save = steps.find((step) => step.uses?.startsWith('actions/cache/save@')); +const publish = steps.find((step) => step.with?.path === '.cache/mutation/native-baseline.json'); +const diagnostics = steps.at(-1); + +// Evaluate the actual YAML conditions over canonical fixture values. These fixtures model +// implicit success(), skipped ancestors and workflow cancellation, not a hosted scheduler. +// https://docs.github.com/en/actions/reference/workflows-and-actions/expressions#status-check-functions +// https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idneeds +// https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-cancellation +function condition(expression, context, successful = true, cancelled = false) { + expression = expression.replace(/^\$\{\{\s*|\s*\}\}$/g, ''); + successful &&= !cancelled; + if (!/\b(?:always|success|failure|cancelled)\(/.test(expression) && !successful) return false; + const program = expression.replace( + /\b(?:inputs|steps|needs|github)\.[\w.-]+/g, + (path) => `get('${path}')`, + ); + return Boolean( + runInNewContext(program, { + get: (path) => path.split('.').reduce((value, key) => value?.[key], context) ?? '', + always: () => true, + success: () => successful, + cancelled: () => cancelled, + hashFiles: () => 'fixture-file-digest', + }), + ); +} + +function actionContext(route, overrides = {}) { + return { + inputs: { full: 'false', publish: 'true', 'parallel-fallback': 'true', ...overrides }, + steps: { + mutation: { outputs: { route } }, + 'pr-key': { outputs: { key: 'exact-key' } }, + 'pr-cache': { outputs: { 'cache-hit': 'false' } }, + }, + }; +} + +function inlineStatus(step, env) { + const program = step.run.split("<<'JS'\n")[1]?.split('\nJS')[0]; + assert.ok(program); + return spawnSync(process.execPath, ['--input-type=module', '-e', program], { + env, + encoding: 'utf8', + timeout: 5000, + }).status; +} + +const needsOf = (job) => [jobs[job].needs ?? []].flat(); +const ancestors = (job) => needsOf(job).flatMap((name) => [name, ...ancestors(name)]); +function selected(name, full, states, { expression = jobs[name].if, cancelled = false } = {}) { + const needs = Object.fromEntries(needsOf(name).map((key) => [key, states[key]])); + const successful = ancestors(name).every((key) => states[key]?.result === 'success'); + return condition(expression, { inputs: { full }, needs }, successful, cancelled); +} + +const result = (state, route = '') => ({ result: state, outputs: { route } }); + +test('parallel routing is opt-in and transports exactly one runner output through action and job', () => { + assert.equal(action.inputs['parallel-fallback'].default, 'false'); + assert.equal(action.outputs.route.value, '${{ steps.mutation.outputs.route }}'); + assert.equal(jobs.ordinary.outputs.route, '${{ steps.mutation.outputs.route }}'); + assert.equal( + jobs.ordinary.steps.at(-1).with['parallel-fallback'], + "${{ github.event_name == 'pull_request' }}", + ); + assert.equal(execution.env.PARALLEL_FALLBACK, '${{ inputs.parallel-fallback }}'); + assert.equal( + execution.run.trim(), + [ + 'if [ "$FORCE_FULL" = true ]; then', + ' node scripts/run-mutations.mjs --full', + ' echo \'route=native\' >> "$GITHUB_OUTPUT"', + 'elif [ "$PARALLEL_FALLBACK" = true ]; then', + ' node scripts/run-mutations.mjs --parallel-fallback', + 'else', + ' node scripts/run-mutations.mjs', + ' echo \'route=native\' >> "$GITHUB_OUTPUT"', + 'fi', + ].join('\n'), + ); + assert.equal(execution['continue-on-error'], undefined); + assert.equal(routeGuard['continue-on-error'], undefined); + assert.deepEqual(routeGuard.env, { ROUTE: '${{ steps.mutation.outputs.route }}' }); + assert.ok(steps.indexOf(routeGuard) > steps.indexOf(execution)); + assert.ok(steps.indexOf(routeGuard) < steps.indexOf(encoder)); +}); + +test('only PRs opt into fallback; main, nonfull manual and merge queue keep the native command', () => { + const optIn = jobs.ordinary.steps.at(-1).with['parallel-fallback']; + for (const event of ['pull_request', 'push', 'workflow_dispatch', 'merge_group']) { + const enabled = condition(optIn, { github: { event_name: event } }); + assert.equal(enabled, event === 'pull_request'); + if (enabled) continue; + const context = actionContext('native', { 'parallel-fallback': String(enabled) }); + assert.equal(condition(routeGuard.if, context), false); + assert.equal(condition(encoder.if, context), true); + assert.equal(selected('plan', false, { ordinary: result('success', 'native') }), false); + } +}); + +test('missing and invalid routes fail the ordinary job instead of passing with all campaigns skipped', () => { + for (const route of ['native', 'full-sharded']) + assert.equal(inlineStatus(routeGuard, { ROUTE: route }), 0); + for (const route of ['', 'unknown', 'NATIVE', 'FULL-SHARDED', 'native\n']) { + assert.notEqual(inlineStatus(routeGuard, { ROUTE: route }), 0); + const states = { ordinary: result('failure', route) }; + assert.equal(selected('plan', false, states), false); + } + assert.notEqual(inlineStatus(routeGuard, {}), 0); +}); + +test('only a successful native route checks encoders and promotes eligible cache results', () => { + for (const route of ['native', 'full-sharded', '', 'unknown']) { + const context = actionContext(route); + assert.equal(condition(routeGuard.if, context), true); + for (const step of [encoder, save, publish]) { + assert.equal(condition(step.if, context), route === 'native'); + assert.equal(condition(step.if, context, false), false); + } + assert.equal(condition(diagnostics.if, context, false), true); + } + assert.equal(diagnostics.with.path, 'reports/mutation/'); + assert.equal(diagnostics['continue-on-error'], undefined); + for (const overrides of [{ 'parallel-fallback': 'false' }, { full: 'true' }]) { + const context = actionContext('', overrides); + assert.equal(condition(routeGuard.if, context), false); + for (const step of [encoder, save, publish]) assert.equal(condition(step.if, context), true); + } + assert.equal(condition(save.if, actionContext('native', { publish: 'false' })), true); + assert.equal(condition(publish.if, actionContext('native', { publish: 'false' })), false); + const cacheHit = actionContext('native'); + cacheHit.steps['pr-cache'].outputs['cache-hit'] = 'true'; + assert.equal(condition(save.if, cacheHit), false); +}); + +test('warm PR and ordinary main stay native, cold PR and explicit full select the matrix', () => { + for (const [name, full, ordinary, launches] of [ + ['warm PR', false, result('success', 'native'), false], + ['ordinary main', false, result('success', 'native'), false], + ['cold PR', false, result('success', 'full-sharded'), true], + ['explicit full', true, result('skipped'), true], + ['warm failure', false, result('failure', 'native'), false], + ['failed handoff', false, result('failure', 'full-sharded'), false], + ['cancelled handoff', false, result('cancelled', 'full-sharded'), false], + ['missing route', false, result('failure'), false], + ]) { + assert.equal(selected('ordinary', full, {}), !full, name); + const states = { ordinary }; + assert.equal(selected('plan', full, states), launches, name); + states.plan = result(launches ? 'success' : 'skipped'); + assert.equal(selected('shard', full, states), launches, name); + states.shard = result(launches ? 'success' : 'skipped'); + assert.equal(selected('collect', full, states), launches, name); + } +}); + +test('the collector runs and fails for every failed or cancelled selected full campaign', () => { + for (const full of [false, true]) { + for (const plan of ['success', 'failure', 'cancelled']) { + for (const shard of ['success', 'failure', 'cancelled', 'skipped']) { + const states = { + ordinary: result(full ? 'skipped' : 'success', full ? '' : 'full-sharded'), + plan: result(plan), + shard: result(shard), + }; + assert.equal(selected('shard', full, states), plan === 'success'); + assert.equal(selected('collect', full, states), true); + const status = inlineStatus(jobs.collect.steps[0], { + RESULTS: JSON.stringify({ plan: states.plan, shard: states.shard }), + }); + assert.equal(status === 0, plan === 'success' && shard === 'success'); + } + } + } +}); + +test('status checks preserve explicit full skipped ancestors and failed-campaign collection', () => { + const states = { ordinary: result('skipped'), plan: result('success'), shard: result('success') }; + for (const name of ['plan', 'shard', 'collect']) { + assert.equal(selected(name, true, states), true); + const expression = jobs[name].if.replace(/(?:!cancelled\(\)|always\(\)) && /, ''); + assert.equal(selected(name, true, states, { expression }), false); + } + states.ordinary = result('success', 'full-sharded'); + states.shard = result('failure'); + assert.equal(selected('collect', false, states), true); + assert.equal( + selected('collect', false, states, { expression: jobs.collect.if.replace('always() && ', '') }), + false, + ); +}); + +test('workflow cancellation stops obsolete plans and shards while collection retains failure evidence', () => { + for (const full of [false, true]) { + const states = { + ordinary: result(full ? 'skipped' : 'success', full ? '' : 'full-sharded'), + plan: result('success'), + shard: result('success'), + }; + for (const name of ['plan', 'shard']) { + assert.equal(selected(name, full, states), true); + assert.equal(selected(name, full, states, { cancelled: true }), false); + // Successful dependencies do not imply the workflow has escaped cancellation. + const expression = jobs[name].if.replace('!cancelled()', 'always()'); + assert.equal(selected(name, full, states, { expression, cancelled: true }), true); + } + states.plan = result('cancelled'); + states.shard = result('skipped'); + assert.equal(selected('collect', full, states, { cancelled: true }), true); + assert.notEqual( + inlineStatus(jobs.collect.steps[0], { + RESULTS: JSON.stringify({ plan: states.plan, shard: states.shard }), + }), + 0, + ); + } +}); diff --git a/scripts/mutation-routing.test.mjs b/scripts/mutation-routing.test.mjs new file mode 100644 index 0000000..e0e787b --- /dev/null +++ b/scripts/mutation-routing.test.mjs @@ -0,0 +1,351 @@ +import assert from 'node:assert/strict'; +import { appendFile, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { runInNewContext } from 'node:vm'; +import { + BASELINE, + PR_BASELINE, + WORKING_REPORT, + executionSummary, + fingerprint, + promoteCache, + pullRequestIdentity, + readCache, +} from './mutation-cache.mjs'; +import { sourceContext } from './mutation-policy.mjs'; + +const commit = 'a'.repeat(40); +const inputs = { files: [['src/mp3/example.ts', 420, 'source-hash']] }; +const key = fingerprint(inputs); +const prEnv = { + GITHUB_ACTIONS: 'true', + GITHUB_EVENT_NAME: 'pull_request', + GITHUB_REF: 'refs/pull/7/merge', + GITHUB_REPOSITORY_ID: '42', + GITHUB_SHA: 'c'.repeat(40), + MUTATION_BASELINE_SHA: commit, +}; +const report = { + valid: true, + files: { 'src/mp3/example.ts': { mutants: [{ status: 'Killed', duration: 0 }] } }, + testFiles: {}, +}; +const summary = { + rawScore: 100, + adjustedScore: 100, + adjustedDetected: 1, + adjustedTotal: 1, + killed: 1, + timedOut: 0, + equivalent: 0, + timedOutEquivalents: [], +}; +const runnerSource = await readFile(new URL('./run-mutations.mjs', import.meta.url), 'utf8'); +// Execute the complete runner body with fake process/discovery boundaries. The +// actual source-proof check, cache admission, filesystem cleanup and finally run. +const runnerBody = runnerSource + .slice(runnerSource.indexOf('const root =')) + .replaceAll('import.meta.url', 'runnerUrl') + .replaceAll('import.meta.resolve(', 'resolve('); + +async function fixture(t, options = {}) { + const root = await mkdtemp(join(tmpdir(), 'mutation-routing-')); + t.after(() => rm(root, { recursive: true, force: true })); + const source = 'export const value = 1;\n'; + await mkdir(join(root, 'src/mp3'), { recursive: true }); + await writeFile(join(root, 'src/mp3/example.ts'), source); + await writeFile( + join(root, 'mutation-equivalents.json'), + JSON.stringify({ + contextSha256: options.staleProof + ? 'stale' + : sourceContext(new Map([['src/mp3/example.ts', source]])), + }), + ); + const outputPath = join(root, 'github-output'); + await writeFile(outputPath, ''); + const env = { ...prEnv, GITHUB_OUTPUT: outputPath, ...options.env }; + const identity = env.MUTATION_PR_CACHE_RESTORED === 'true' ? pullRequestIdentity(env) : commit; + const baselinePath = join(root, identity?.startsWith('pr:') ? PR_BASELINE : BASELINE); + if (options.cache) { + await promoteCache(root, report, key, identity); + const envelope = JSON.parse(await readFile(baselinePath, 'utf8')); + if (options.cache === 'incompatible') envelope.key = 'f'.repeat(64); + if (options.cache === 'unauditable') envelope.report = { valid: false }; + await writeFile( + baselinePath, + options.cache === 'corrupt' ? '{partial' : JSON.stringify(envelope), + ); + } + const calls = { discovery: 0, processes: [], promotions: [], ended: 0 }; + const messages = []; + const processState = { + argv: ['node', 'run-mutations.mjs', ...(options.args ?? ['--parallel-fallback'])], + env, + }; + const context = { + URL, + runnerUrl: pathToFileURL(join(root, 'scripts/run-mutations.mjs')).href, + resolve: (specifier) => pathToFileURL(join(root, 'node_modules', specifier)).href, + fileURLToPath, + dirname, + join, + relative, + readFile, + readdir, + rm, + mkdir, + writeFile: async (path, ...args) => { + if (options.diagnosticFailure && path.endsWith('routing.json')) + throw new Error('diagnostic unavailable'); + await writeFile(path, ...args); + }, + appendFile: async (path, bytes) => { + const routing = JSON.parse( + await readFile(join(root, 'reports/mutation/routing.json'), 'utf8'), + ); + assert.equal(bytes, `route=${routing.route}\n`); + if (options.outputFailure) throw new Error('output unavailable'); + await appendFile(path, bytes); + }, + createWriteStream: () => ({ end: () => calls.ended++ }), + process: processState, + performance, + console: Object.fromEntries( + ['log', 'warn', 'error'].map((method) => [method, (message) => messages.push(message)]), + ), + BASELINE, + PR_BASELINE, + WORKING_REPORT, + mutationEnvironment: () => (options.nodeOptions ? { NODE_OPTIONS: '--inspect=0' } : {}), + mutationInputs: async () => { + if (options.inputFailure) throw new Error('input unavailable'); + return inputs; + }, + fingerprint, + sourceContext, + pullRequestIdentity, + readCache, + executionSummary, + auditMutations: (value) => { + if (!value?.valid) throw new Error('audit rejected report'); + return summary; + }, + verifyParserTestDiscovery: async () => { + calls.discovery++; + return { selected: ['tests/mp3/example.test.ts'] }; + }, + runMutationProcess: async (command, args) => { + calls.processes.push({ command, args: Array.from(args) }); + if (options.processFailure) throw new Error('native execution failed'); + if (command.endsWith('stryker.js')) + await writeFile(join(root, 'reports/mutation/mutation.json'), JSON.stringify(report)); + }, + promoteCache: async (...values) => { + calls.promotions.push(values[3]); + await promoteCache(...values); + }, + }; + await runInNewContext(`(async () => {${runnerBody}})()`, context, { timeout: 1000 }); + await assert.rejects(readFile(join(root, '.cache/mutation/run.lock')), { code: 'ENOENT' }); + return { + root, + calls, + messages, + exitCode: processState.exitCode ?? 0, + route: await readFile(outputPath, 'utf8'), + }; +} + +for (const [cache, reason] of [ + [undefined, 'missing'], + ['incompatible', 'incompatible'], + ['corrupt', 'corrupt-or-unauditable'], + ['unauditable', 'corrupt-or-unauditable'], +]) + test(`cold hosted PR routes before discovery for a ${cache ?? 'missing'} baseline`, async (t) => { + const result = await fixture(t, { cache }); + assert.equal(result.exitCode, 0); + assert.equal(result.route, 'route=full-sharded\n'); + assert.equal(result.calls.discovery, 0); + assert.deepEqual(result.calls.processes, []); + assert.deepEqual(result.calls.promotions, []); + assert.equal(result.calls.ended, 1); + assert.ok(result.messages.some((message) => message.includes(`native-baseline-${reason}.`))); + assert.deepEqual( + JSON.parse(await readFile(join(result.root, 'reports/mutation/routing.json'), 'utf8')), + { + route: 'full-sharded', + cacheReason: `native-baseline-${reason}`, + inputsHash: key, + inputCaptureError: null, + }, + ); + await assert.rejects(readFile(join(result.root, 'reports/mutation/audit.json')), { + code: 'ENOENT', + }); + await assert.rejects(readFile(join(result.root, BASELINE)), { code: 'ENOENT' }); + await assert.rejects(readFile(join(result.root, WORKING_REPORT)), { code: 'ENOENT' }); + }); + +for (const options of [ + { inputFailure: true }, + { nodeOptions: true, cache: 'valid' }, + { env: { GITHUB_HEAD_REPOSITORY_FORK: 'true' } }, +]) + test(`PR routing remains eligible for ${JSON.stringify(options)}`, async (t) => { + const result = await fixture(t, options); + assert.equal(result.exitCode, 0); + assert.equal(result.route, 'route=full-sharded\n'); + assert.equal(result.calls.discovery, 0); + assert.deepEqual(result.calls.processes, []); + assert.deepEqual( + JSON.parse(await readFile(join(result.root, 'reports/mutation/routing.json'), 'utf8')), + { + route: 'full-sharded', + cacheReason: + options.inputFailure || options.nodeOptions + ? 'native-full-required' + : 'native-baseline-missing', + inputsHash: options.inputFailure ? null : key, + inputCaptureError: options.inputFailure ? 'unsupported-input' : null, + }, + ); + }); + +for (const restored of ['false', 'true']) + test(`warm PR keeps native reuse with own-PR restore=${restored}`, async (t) => { + const result = await fixture(t, { + cache: 'valid', + env: { MUTATION_PR_CACHE_RESTORED: restored }, + }); + assert.equal(result.exitCode, 0); + assert.equal(result.route, 'route=native\n'); + assert.deepEqual( + JSON.parse(await readFile(join(result.root, 'reports/mutation/routing.json'), 'utf8')), + { + route: 'native', + cacheReason: 'native-same-input-reuse', + inputsHash: key, + inputCaptureError: null, + }, + ); + assert.equal(result.calls.discovery, 1); + assert.deepEqual( + result.calls.processes.map(({ args }) => args), + [['run', '--no-fsModuleCache'], ['run']], + ); + assert.deepEqual(result.calls.promotions, ['pr:42:7']); + assert.deepEqual(JSON.parse(await readFile(join(result.root, WORKING_REPORT), 'utf8')), report); + }); + +test('cold main push runs native full and promotes the trusted sequential report', async (t) => { + const result = await fixture(t, { + env: { GITHUB_EVENT_NAME: 'push', GITHUB_REF: 'refs/heads/main', GITHUB_SHA: commit }, + }); + assert.equal(result.exitCode, 0); + assert.equal(result.route, 'route=native\n'); + assert.equal(result.calls.discovery, 1); + assert.deepEqual( + result.calls.processes.map(({ args }) => args), + [ + ['run', '--no-fsModuleCache'], + ['run', '--force'], + ], + ); + assert.deepEqual(result.calls.promotions, [commit]); +}); + +for (const env of [ + { GITHUB_ACTIONS: '' }, + { GITHUB_EVENT_NAME: 'pull_request_target' }, + { GITHUB_EVENT_NAME: 'workflow_dispatch' }, + { GITHUB_REF: 'refs/pull/7/head' }, + { GITHUB_REPOSITORY_ID: '' }, + { GITHUB_OUTPUT: '' }, +]) + test(`invalid routing context fails before execution: ${JSON.stringify(env)}`, async (t) => { + const result = await fixture(t, { env }); + assert.equal(result.exitCode, 1); + assert.equal(result.route, ''); + assert.equal(result.calls.discovery, 0); + assert.deepEqual(result.calls.processes, []); + assert.deepEqual(result.calls.promotions, []); + }); + +for (const options of [ + { outputFailure: true }, + { outputFailure: true, cache: 'valid' }, + { diagnosticFailure: true }, + { staleProof: true }, +]) + test(`routing prerequisite failure stops without a route: ${JSON.stringify(options)}`, async (t) => { + const result = await fixture(t, options); + assert.equal(result.exitCode, 1); + assert.equal(result.route, ''); + assert.equal(result.calls.discovery, 0); + assert.deepEqual(result.calls.processes, []); + assert.deepEqual(result.calls.promotions, []); + assert.equal(result.calls.ended, 1); + }); + +test('warm execution failure cannot fall back to sharded execution', async (t) => { + const result = await fixture(t, { cache: 'valid', processFailure: true }); + assert.equal(result.exitCode, 1); + assert.equal(result.route, 'route=native\n'); + assert.equal(result.calls.discovery, 1); + assert.equal(result.calls.processes.length, 1); + assert.deepEqual(result.calls.promotions, []); +}); + +test('standalone full and cache-key preserve their behavior without routing output', async (t) => { + const full = await fixture(t, { args: ['--full'], cache: 'valid' }); + assert.equal(full.exitCode, 0); + assert.equal(full.route, ''); + assert.equal(full.calls.discovery, 1); + assert.deepEqual(full.calls.processes[1].args, ['run', '--force']); + assert.deepEqual(full.calls.promotions, []); + const cacheKey = await fixture(t, { args: ['--cache-key'] }); + assert.equal(cacheKey.exitCode, 0); + assert.equal(cacheKey.route, ''); + assert.deepEqual(cacheKey.messages, [key]); + assert.equal(cacheKey.calls.discovery, 0); + assert.equal(cacheKey.calls.ended, 0); + assert.deepEqual(cacheKey.calls.processes, []); +}); + +test('the ordinary non-opt-in cold runner still executes native full', async (t) => { + const result = await fixture(t, { args: [] }); + assert.equal(result.exitCode, 0); + assert.equal(result.route, ''); + assert.equal(result.calls.discovery, 1); + assert.deepEqual(result.calls.processes[1].args, ['run', '--force']); + assert.deepEqual(result.calls.promotions, ['pr:42:7']); +}); + +test('parallel fallback cannot be combined with standalone modes or scope arguments', async (t) => { + for (const args of [ + ['--parallel-fallback', '--full'], + ['--parallel-fallback', '--cache-key'], + ['--parallel-fallback', '--mutate=src/mp3/example.ts'], + ]) { + const result = await fixture(t, { args }); + assert.equal(result.exitCode, 1); + assert.equal(result.route, ''); + assert.equal(result.calls.discovery, 0); + assert.deepEqual(result.calls.processes, []); + } +}); + +test('unreadable baseline diagnostics remain a miss and do not call the report audit', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'mutation-unreadable-cache-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, BASELINE), { recursive: true }); + const result = await readCache(root, key, commit, () => + assert.fail('must not audit unreadable data'), + ); + assert.deepEqual(result, { report: null, reason: 'native-baseline-unreadable' }); +}); diff --git a/scripts/mutation-workflow.test.mjs b/scripts/mutation-workflow.test.mjs index c51f4c4..652fe20 100644 --- a/scripts/mutation-workflow.test.mjs +++ b/scripts/mutation-workflow.test.mjs @@ -21,12 +21,19 @@ test('ordinary mutation keeps the existing native action and environment without assert.equal(ordinary.strategy, undefined); assert.equal(ordinary['runs-on'], 'ubuntu-24.04'); assert.equal(ordinary['timeout-minutes'], 100); + assert.deepEqual(ordinary.outputs, { route: '${{ steps.mutation.outputs.route }}' }); assert.deepEqual(ordinary.steps.slice(1), [ { uses: checkout, with: { ref: '${{ github.sha }}', 'persist-credentials': false } }, { uses: './.github/actions/setup-project' }, { uses: './.github/actions/test-mutations', - with: { base: '${{ inputs.base }}', full: 'false', publish: '${{ inputs.publish }}' }, + id: 'mutation', + with: { + base: '${{ inputs.base }}', + full: 'false', + publish: '${{ inputs.publish }}', + 'parallel-fallback': "${{ github.event_name == 'pull_request' }}", + }, }, ]); }); @@ -61,8 +68,12 @@ test('entry jobs reject source drift before checkout or repository code runs', ( }); test('full partitions use a native dynamic matrix capped at eleven beside the pure campaign', () => { - assert.equal(plan.if, 'inputs.full'); - assert.equal(shard.if, 'inputs.full'); + assert.equal( + plan.if, + "${{ !cancelled() && (inputs.full || (needs.ordinary.result == 'success' && needs.ordinary.outputs.route == 'full-sharded')) }}", + ); + assert.equal(plan.needs, 'ordinary'); + assert.equal(shard.if, "${{ !cancelled() && needs.plan.result == 'success' }}"); assert.equal(shard.needs, 'plan'); assert.deepEqual(shard.strategy, { 'fail-fast': false, @@ -112,7 +123,7 @@ test('all branches check out the exact caller source and keep pinned read-only d }); test('collection requires successful dependencies before downloading current-attempt evidence', () => { - assert.equal(collect.if, 'always() && inputs.full'); + assert.equal(collect.if, "always() && needs.plan.result != 'skipped'"); assert.deepEqual(collect.needs, ['plan', 'shard']); const guard = collect.steps[0]; assert.deepEqual(guard.env, { RESULTS: '${{ toJSON(needs) }}' }); diff --git a/scripts/run-mutations.mjs b/scripts/run-mutations.mjs index 74fb9f0..c8abaf0 100644 --- a/scripts/run-mutations.mjs +++ b/scripts/run-mutations.mjs @@ -1,6 +1,6 @@ import { createWriteStream } from 'node:fs'; import { runMutationProcess } from './mutation-process.mjs'; -import { readFile, readdir, rm, mkdir, writeFile } from 'node:fs/promises'; +import { appendFile, readFile, readdir, rm, mkdir, writeFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import { fileURLToPath } from 'node:url'; import { auditMutations, sourceContext } from './mutation-policy.mjs'; @@ -37,11 +37,14 @@ async function parserSources(directory = join(root, 'src/mp3')) { let log; let locked = false; const lock = join(root, '.cache/mutation/run.lock'); -try { +mutationRun: try { const args = process.argv.slice(2); - if (args.some((arg) => arg !== '--full' && arg !== '--cache-key') || args.length > 1) { + if ( + args.some((arg) => !['--full', '--cache-key', '--parallel-fallback'].includes(arg)) || + args.length > 1 + ) { throw new Error( - 'Only --full and --cache-key are supported; narrowing mutation scope is forbidden', + 'Only --full, --cache-key or --parallel-fallback is supported; narrowing mutation scope is forbidden', ); } if (!args.includes('--cache-key')) { @@ -82,11 +85,10 @@ try { 'Parser context changed: review equivalence proofs before rerunning mutations', ); } - const discovery = await verifyParserTestDiscovery(root, env); - await writeFile(join(output, 'discovery.json'), JSON.stringify(discovery, null, 2) + '\n'); const audit = (report, options) => auditMutations(report, policy, sources, options); const force = args.includes('--full') || Boolean(env.NODE_OPTIONS) || !inputsHash; - const prIdentity = force ? null : pullRequestIdentity(process.env); + const routingIdentity = pullRequestIdentity(process.env); + const prIdentity = force ? null : routingIdentity; const sourceIdentity = prIdentity && process.env.MUTATION_PR_CACHE_RESTORED === 'true' ? prIdentity @@ -103,6 +105,29 @@ try { if (process.env.GITHUB_ACTIONS === 'true') for (const path of [BASELINE, PR_BASELINE]) await rm(join(root, path), { force: true }); console.log(`Mutation cache: ${cache.reason}. Native Stryker owns result reuse.`); + if (args.includes('--parallel-fallback')) { + const mainPush = + process.env.GITHUB_ACTIONS === 'true' && + process.env.GITHUB_EVENT_NAME === 'push' && + process.env.GITHUB_REF === 'refs/heads/main'; + if (!routingIdentity && !mainPush) + throw new Error('Parallel fallback requires a hosted pull-request merge or main push'); + if (!process.env.GITHUB_OUTPUT) throw new Error('Parallel fallback requires GITHUB_OUTPUT'); + const route = routingIdentity && !cache.report ? 'full-sharded' : 'native'; + await writeFile( + join(output, 'routing.json'), + JSON.stringify( + { route, cacheReason: cache.reason, inputsHash, inputCaptureError }, + null, + 2, + ) + '\n', + ); + await appendFile(process.env.GITHUB_OUTPUT, `route=${route}\n`); + // Leave through finally so the shard handoff never retains the local lock. + if (route === 'full-sharded') break mutationRun; + } + const discovery = await verifyParserTestDiscovery(root, env); + await writeFile(join(output, 'discovery.json'), JSON.stringify(discovery, null, 2) + '\n'); const started = performance.now(); // A normal unmutated run is mandatory even for a 100% cache hit. Stryker // subsequently performs its own fresh discovery/coverage dry run as well.