From 3638edbab443d63260d632e497a67c2f6393692f Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 03:49:38 +0000 Subject: [PATCH 1/2] ci(repo): run the mutation lane on pull requests and gate new survivors A pull request runs the same preflight and sharded incremental run against main's restored verdict cache, saves no cache and records no timings, then gates its merged report against the survivor baseline main published. A run that evaluated no mutants says so in a notice instead of reading as a pass Verdict-Semantics: unchanged --- .github/workflows/mutation.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 43817f080..b8f9b5a40 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -3,6 +3,7 @@ name: Mutation on: push: branches: [main] + pull_request: workflow_dispatch: concurrency: @@ -223,7 +224,7 @@ jobs: timings: name: record mutation timings needs: [plan, mutation] - if: ${{ always() && needs.plan.result == 'success' }} + if: ${{ always() && needs.plan.result == 'success' && github.event_name != 'pull_request' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -280,7 +281,7 @@ jobs: path: .stryker-incremental merge-multiple: true - uses: actions/cache/save@v6 - if: ${{ hashFiles('.stryker-incremental/**') != '' }} + if: ${{ github.event_name != 'pull_request' && hashFiles('.stryker-incremental/**') != '' }} with: path: .stryker-incremental key: stryker-incremental-${{ github.run_id }} @@ -329,6 +330,28 @@ jobs: path: mutation-gate/mutation-survivor-baseline.json retention-days: 90 + - name: Survivor baseline from main + if: ${{ github.event_name == 'pull_request' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p mutation-gate/reports/mutation + id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=mutation-survivor-baseline&per_page=100" \ + --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main")][0].id // empty') || id= + if [ -n "$id" ] && gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip" > mutation-gate/baseline.zip; then + unzip -o -q mutation-gate/baseline.zip -d mutation-gate + fi + - name: Gate new survivors + if: ${{ github.event_name == 'pull_request' }} + run: | + if [ ! -f mutation-gate/mutation-survivor-baseline.json ]; then + echo "::warning title=Mutation gate skipped::main has published no survivor baseline yet, so this run cannot tell a new survivor from an accepted one" + exit 0 + fi + cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json + cd mutation-gate + "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json + - name: Delete per-job artifacts if: ${{ always() && needs.mutation.result == 'success' }} continue-on-error: true From bda220ac3a750aa2149e54cadc3b77fd372f894d Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 04:46:25 +0000 Subject: [PATCH 2/2] ci(repo): plan mutation shards from the full mutant set's cost This stack's own pull request ran 7465 invalidated stryker-js mutants in one job planned at 9m43s and hit the 1800s cap: the timing record held main's reuse-shrunk duration. Each entry now carries the reuse line's ran and reused counts, the record scales a run to the full mutant set and keeps its previous value when nothing ran, and shards download the preflight coverage only when the preflight published some Verdict-Semantics: unchanged --- .github/workflows/mutation.yml | 7 +++++-- scripts/lib/mutation-plan.test.ts | 9 +++++++++ scripts/lib/mutation-plan.ts | 32 +++++++++++++++++++++---------- scripts/mutation-job.ts | 15 ++++++++++----- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index b8f9b5a40..f4f853171 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -58,6 +58,8 @@ jobs: if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} runs-on: ubuntu-latest timeout-minutes: 45 + outputs: + coverage: ${{ steps.preflight.outputs.coverage }} env: OTEL_ENABLED: "true" OTEL_SERVICE_NAME: stryker-js-mutation @@ -94,7 +96,8 @@ jobs: echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" pnpm lgtm:up - - name: Preflight + - id: preflight + name: Preflight env: JOBS: ${{ needs.plan.outputs.jobs }} run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS" @@ -164,7 +167,7 @@ jobs: - name: Place incremental files run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi - uses: actions/download-artifact@v8 - continue-on-error: true + if: ${{ needs.preflight.outputs.coverage == 'true' }} with: name: mutation-preflight path: . diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index ed3d3abc8..50fb525b2 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -329,3 +329,12 @@ Deno.test('mergeSarif folds every package into one run with repository paths and ) assertEquals(mergeSarif([]), Option.none()) }) + +Deno.test('mergeRecord scales a reuse-shrunk duration to the full mutant set and keeps the record when nothing ran', () => { + const previous: TimingRecord = { version: 1, packages: { p: { seconds: 500, sha: 'old' } } } + const scaled: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 100, exitCode: 0, ran: 10, reused: 90 }] }] + assertEquals(mergeRecord(previous, scaled, 'new').packages['p'], { seconds: 1000, sha: 'new' }) + const nothing: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 20, exitCode: 0, ran: 0, reused: 100 }] }] + assertEquals(mergeRecord(previous, nothing, 'new').packages['p'], { seconds: 500, sha: 'old' }) + assertEquals(mergeRecord(emptyRecord, nothing, 'new').packages['p'], { seconds: 20, sha: 'new' }) +}) diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 24fe1ff34..86ddea670 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -29,6 +29,8 @@ export const EntrySchema = S.Struct({ seconds: S.Number, exitCode: S.Union([S.Int, S.Null]), shard: S.optional(ShardSchema), + ran: S.optional(S.Int), + reused: S.optional(S.Int), }) export type Entry = S.Schema.Type @@ -176,6 +178,19 @@ export const planJobs = ( return { jobs: [...shardJobs, ...wholeJobs] } } +const fullSetSecondsOf = (entry: Entry): number => + entry.ran === undefined || entry.reused === undefined || entry.ran === 0 + ? entry.seconds + : Math.round((entry.seconds * (entry.ran + entry.reused)) / entry.ran) + +const measuredOf = (entries: readonly Entry[]): number | undefined => { + const count = entries[0]?.shard?.count + if (count === undefined) return Math.max(...entries.map(fullSetSecondsOf)) + const indices = new Set(entries.map((entry) => entry.shard?.index)) + const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) + return complete ? entries.reduce((sum, entry) => sum + fullSetSecondsOf(entry), 0) : undefined +} + export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: string): TimingRecord => { const packages: Record = { ...previous.packages } const byPackage = new Map() @@ -183,14 +198,10 @@ export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: byPackage.set(entry.package, [...(byPackage.get(entry.package) ?? []), entry]) } for (const [name, entries] of byPackage) { - const count = entries[0]?.shard?.count - if (count === undefined) { - packages[name] = { seconds: Math.max(...entries.map((entry) => entry.seconds)), sha } - continue - } - const indices = new Set(entries.map((entry) => entry.shard?.index)) - const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) - if (complete) packages[name] = { seconds: entries.reduce((sum, entry) => sum + entry.seconds, 0), sha } + const evaluatedNothing = entries.some((entry) => entry.ran === 0) + const seconds = measuredOf(entries) + if (seconds === undefined || (evaluatedNothing && previous.packages[name] !== undefined)) continue + packages[name] = { seconds, sha } } return { version: 1, packages } } @@ -241,6 +252,7 @@ const RunFailedLineSchema = S.Struct({ const ReuseLineSchema = S.Struct({ _tag: S.Literal('reuse'), ran: S.Int.check(S.isGreaterThanOrEqualTo(0)), + reused: S.Int.check(S.isGreaterThanOrEqualTo(0)), }) const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder): ReadonlyArray => @@ -251,10 +263,10 @@ const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder< const terminalRecordOf = (streamText: string | null): Option.Option => Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record) -export type ReuseState = { readonly ran: number } +export type ReuseState = { readonly ran: number; readonly reused: number } const reuseOf = (streamText: string | null): Option.Option => - Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran })) + Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran, reused: line.reused })) export type ReportState = { reportText: string | null diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index eddb9bee6..fddd5716f 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -106,12 +106,18 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`) ok = false } + const seconds = Math.round((Date.now() - started) / 1000) + const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' + const reportsDir = join(dir, 'reports') + const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } + const state = await loadState(reportsDir, readText) if (exitCode !== null) { entries.push({ package: name, - seconds: Math.round((Date.now() - started) / 1000), + seconds, exitCode, ...(shard === undefined ? {} : { shard }), + ...Option.match(state.reuse, { onNone: () => ({}), onSome: ({ ran, reused }) => ({ ran, reused }) }), }) await Deno.mkdir('.timings', { recursive: true }) await Deno.writeTextFile( @@ -119,10 +125,6 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom JSON.stringify({ job: job.id, entries } satisfies Part), ) } - const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' - const reportsDir = join(dir, 'reports') - const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } - const state = await loadState(reportsDir, readText) console.log(buildSummary(input, state)) const missing = buildRequireError(input, state) if (missing !== null) { @@ -178,9 +180,12 @@ const preflightPackage = async (name: string, dir: string, capSeconds: number): const preflight = async (jobs: readonly Job[], capSeconds: number): Promise => { let ok = true + let coverage = false for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) { if (!await preflightPackage(name, dir, capSeconds)) ok = false + if ((await readIfPresent(join('preflight', dir, PREFLIGHT_FILE))) !== undefined) coverage = true } + await appendTo('GITHUB_OUTPUT', `coverage=${coverage}\n`) return ok }