ci(repo): run the mutation lane on pull requests and gate new survivors

.github/workflows/mutation.yml

@@ -3,6 +3,7 @@ name: Mutation
 on:
   push:
     branches: [main]
+  pull_request:
   workflow_dispatch:
 
 concurrency:
@@ -57,6 +58,8 @@ jobs:
     if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }}
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    outputs:
+      coverage: ${{ steps.preflight.outputs.coverage }}
     env:
       OTEL_ENABLED: "true"
       OTEL_SERVICE_NAME: stryker-js-mutation
@@ -93,7 +96,8 @@ jobs:
           echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV"
           pnpm lgtm:up
 
-      - name: Preflight
+      - id: preflight
+        name: Preflight
         env:
           JOBS: ${{ needs.plan.outputs.jobs }}
         run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS"
@@ -163,7 +167,7 @@ jobs:
       - name: Place incremental files
         run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi
       - uses: actions/download-artifact@v8
-        continue-on-error: true
+        if: ${{ needs.preflight.outputs.coverage == 'true' }}
         with:
           name: mutation-preflight
           path: .
@@ -223,7 +227,7 @@ jobs:
   timings:
     name: record mutation timings
     needs: [plan, mutation]
-    if: ${{ always() && needs.plan.result == 'success' }}
+    if: ${{ always() && needs.plan.result == 'success' && github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -280,7 +284,7 @@ jobs:
           path: .stryker-incremental
           merge-multiple: true
       - uses: actions/cache/save@v6
-        if: ${{ hashFiles('.stryker-incremental/**') != '' }}
+        if: ${{ github.event_name != 'pull_request' && hashFiles('.stryker-incremental/**') != '' }}
         with:
           path: .stryker-incremental
           key: stryker-incremental-${{ github.run_id }}
@@ -329,6 +333,28 @@ jobs:
           path: mutation-gate/mutation-survivor-baseline.json
           retention-days: 90
 
+      - name: Survivor baseline from main
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p mutation-gate/reports/mutation
+          id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=mutation-survivor-baseline&per_page=100" \
+            --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main")][0].id // empty') || id=
+          if [ -n "$id" ] && gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip" > mutation-gate/baseline.zip; then
+            unzip -o -q mutation-gate/baseline.zip -d mutation-gate
+          fi
+      - name: Gate new survivors
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          if [ ! -f mutation-gate/mutation-survivor-baseline.json ]; then
+            echo "::warning title=Mutation gate skipped::main has published no survivor baseline yet, so this run cannot tell a new survivor from an accepted one"
+            exit 0
+          fi
+          cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json
+          cd mutation-gate
+          "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json
+
       - name: Delete per-job artifacts
         if: ${{ always() && needs.mutation.result == 'success' }}
         continue-on-error: true

scripts/lib/mutation-plan.test.ts

@@ -329,3 +329,12 @@ Deno.test('mergeSarif folds every package into one run with repository paths and
   )
   assertEquals(mergeSarif([]), Option.none())
 })
+
+Deno.test('mergeRecord scales a reuse-shrunk duration to the full mutant set and keeps the record when nothing ran', () => {
+  const previous: TimingRecord = { version: 1, packages: { p: { seconds: 500, sha: 'old' } } }
+  const scaled: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 100, exitCode: 0, ran: 10, reused: 90 }] }]
+  assertEquals(mergeRecord(previous, scaled, 'new').packages['p'], { seconds: 1000, sha: 'new' })
+  const nothing: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 20, exitCode: 0, ran: 0, reused: 100 }] }]
+  assertEquals(mergeRecord(previous, nothing, 'new').packages['p'], { seconds: 500, sha: 'old' })
+  assertEquals(mergeRecord(emptyRecord, nothing, 'new').packages['p'], { seconds: 20, sha: 'new' })
+})

scripts/lib/mutation-plan.ts

@@ -29,6 +29,8 @@ export const EntrySchema = S.Struct({
   seconds: S.Number,
   exitCode: S.Union([S.Int, S.Null]),
   shard: S.optional(ShardSchema),
+  ran: S.optional(S.Int),
+  reused: S.optional(S.Int),
 })
 export type Entry = S.Schema.Type<typeof EntrySchema>
 
@@ -176,21 +178,30 @@ export const planJobs = (
   return { jobs: [...shardJobs, ...wholeJobs] }
 }
 
+const fullSetSecondsOf = (entry: Entry): number =>
+  entry.ran === undefined || entry.reused === undefined || entry.ran === 0
+    ? entry.seconds
+    : Math.round((entry.seconds * (entry.ran + entry.reused)) / entry.ran)
+
+const measuredOf = (entries: readonly Entry[]): number | undefined => {
+  const count = entries[0]?.shard?.count
+  if (count === undefined) return Math.max(...entries.map(fullSetSecondsOf))
+  const indices = new Set(entries.map((entry) => entry.shard?.index))
+  const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index))
+  return complete ? entries.reduce((sum, entry) => sum + fullSetSecondsOf(entry), 0) : undefined
+}
+
 export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: string): TimingRecord => {
   const packages: Record<string, Measured> = { ...previous.packages }
   const byPackage = new Map<string, Entry[]>()
   for (const entry of parts.flatMap((part) => part.entries)) {
     byPackage.set(entry.package, [...(byPackage.get(entry.package) ?? []), entry])
   }
   for (const [name, entries] of byPackage) {
-    const count = entries[0]?.shard?.count
-    if (count === undefined) {
-      packages[name] = { seconds: Math.max(...entries.map((entry) => entry.seconds)), sha }
-      continue
-    }
-    const indices = new Set(entries.map((entry) => entry.shard?.index))
-    const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index))
-    if (complete) packages[name] = { seconds: entries.reduce((sum, entry) => sum + entry.seconds, 0), sha }
+    const evaluatedNothing = entries.some((entry) => entry.ran === 0)
+    const seconds = measuredOf(entries)
+    if (seconds === undefined || (evaluatedNothing && previous.packages[name] !== undefined)) continue
+    packages[name] = { seconds, sha }
   }
   return { version: 1, packages }
 }
@@ -241,6 +252,7 @@ const RunFailedLineSchema = S.Struct({
 const ReuseLineSchema = S.Struct({
   _tag: S.Literal('reuse'),
   ran: S.Int.check(S.isGreaterThanOrEqualTo(0)),
+  reused: S.Int.check(S.isGreaterThanOrEqualTo(0)),
 })
 
 const decodedLines = <A>(streamText: string | null, schema: S.ConstraintDecoder<A>): ReadonlyArray<A> =>
@@ -251,10 +263,10 @@ const decodedLines = <A>(streamText: string | null, schema: S.ConstraintDecoder<
 const terminalRecordOf = (streamText: string | null): Option.Option<FailureRecordType> =>
   Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record)
 
-export type ReuseState = { readonly ran: number }
+export type ReuseState = { readonly ran: number; readonly reused: number }
 
 const reuseOf = (streamText: string | null): Option.Option<ReuseState> =>
-  Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran }))
+  Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran, reused: line.reused }))
 
 export type ReportState = {
   reportText: string | null

scripts/mutation-job.ts

@@ -106,23 +106,25 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom
       console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`)
       ok = false
     }
+    const seconds = Math.round((Date.now() - started) / 1000)
+    const outcome: Outcome = exitCode === 0 ? 'success' : 'failure'
+    const reportsDir = join(dir, 'reports')
+    const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap }
+    const state = await loadState(reportsDir, readText)
     if (exitCode !== null) {
       entries.push({
         package: name,
-        seconds: Math.round((Date.now() - started) / 1000),
+        seconds,
         exitCode,
         ...(shard === undefined ? {} : { shard }),
+        ...Option.match(state.reuse, { onNone: () => ({}), onSome: ({ ran, reused }) => ({ ran, reused }) }),
       })
       await Deno.mkdir('.timings', { recursive: true })
       await Deno.writeTextFile(
         join('.timings', `${job.id}.json`),
         JSON.stringify({ job: job.id, entries } satisfies Part),
       )
     }
-    const outcome: Outcome = exitCode === 0 ? 'success' : 'failure'
-    const reportsDir = join(dir, 'reports')
-    const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap }
-    const state = await loadState(reportsDir, readText)
     console.log(buildSummary(input, state))
     const missing = buildRequireError(input, state)
     if (missing !== null) {
@@ -178,9 +180,12 @@ const preflightPackage = async (name: string, dir: string, capSeconds: number):
 
 const preflight = async (jobs: readonly Job[], capSeconds: number): Promise<boolean> => {
   let ok = true
+  let coverage = false
   for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) {
     if (!await preflightPackage(name, dir, capSeconds)) ok = false
+    if ((await readIfPresent(join('preflight', dir, PREFLIGHT_FILE))) !== undefined) coverage = true
   }
+  await appendTo('GITHUB_OUTPUT', `coverage=${coverage}\n`)
   return ok
 }