diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 43817f080..f4f853171 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -3,6 +3,7 @@ name: Mutation on: push: branches: [main] + pull_request: workflow_dispatch: concurrency: @@ -57,6 +58,8 @@ jobs: if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} runs-on: ubuntu-latest timeout-minutes: 45 + outputs: + coverage: ${{ steps.preflight.outputs.coverage }} env: OTEL_ENABLED: "true" OTEL_SERVICE_NAME: stryker-js-mutation @@ -93,7 +96,8 @@ jobs: echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" pnpm lgtm:up - - name: Preflight + - id: preflight + name: Preflight env: JOBS: ${{ needs.plan.outputs.jobs }} run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS" @@ -163,7 +167,7 @@ jobs: - name: Place incremental files run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi - uses: actions/download-artifact@v8 - continue-on-error: true + if: ${{ needs.preflight.outputs.coverage == 'true' }} with: name: mutation-preflight path: . @@ -223,7 +227,7 @@ jobs: timings: name: record mutation timings needs: [plan, mutation] - if: ${{ always() && needs.plan.result == 'success' }} + if: ${{ always() && needs.plan.result == 'success' && github.event_name != 'pull_request' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -280,7 +284,7 @@ jobs: path: .stryker-incremental merge-multiple: true - uses: actions/cache/save@v6 - if: ${{ hashFiles('.stryker-incremental/**') != '' }} + if: ${{ github.event_name != 'pull_request' && hashFiles('.stryker-incremental/**') != '' }} with: path: .stryker-incremental key: stryker-incremental-${{ github.run_id }} @@ -329,6 +333,28 @@ jobs: path: mutation-gate/mutation-survivor-baseline.json retention-days: 90 + - name: Survivor baseline from main + if: ${{ github.event_name == 'pull_request' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p mutation-gate/reports/mutation + id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=mutation-survivor-baseline&per_page=100" \ + --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main")][0].id // empty') || id= + if [ -n "$id" ] && gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip" > mutation-gate/baseline.zip; then + unzip -o -q mutation-gate/baseline.zip -d mutation-gate + fi + - name: Gate new survivors + if: ${{ github.event_name == 'pull_request' }} + run: | + if [ ! -f mutation-gate/mutation-survivor-baseline.json ]; then + echo "::warning title=Mutation gate skipped::main has published no survivor baseline yet, so this run cannot tell a new survivor from an accepted one" + exit 0 + fi + cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json + cd mutation-gate + "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json + - name: Delete per-job artifacts if: ${{ always() && needs.mutation.result == 'success' }} continue-on-error: true diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index ed3d3abc8..50fb525b2 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -329,3 +329,12 @@ Deno.test('mergeSarif folds every package into one run with repository paths and ) assertEquals(mergeSarif([]), Option.none()) }) + +Deno.test('mergeRecord scales a reuse-shrunk duration to the full mutant set and keeps the record when nothing ran', () => { + const previous: TimingRecord = { version: 1, packages: { p: { seconds: 500, sha: 'old' } } } + const scaled: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 100, exitCode: 0, ran: 10, reused: 90 }] }] + assertEquals(mergeRecord(previous, scaled, 'new').packages['p'], { seconds: 1000, sha: 'new' }) + const nothing: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 20, exitCode: 0, ran: 0, reused: 100 }] }] + assertEquals(mergeRecord(previous, nothing, 'new').packages['p'], { seconds: 500, sha: 'old' }) + assertEquals(mergeRecord(emptyRecord, nothing, 'new').packages['p'], { seconds: 20, sha: 'new' }) +}) diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 24fe1ff34..86ddea670 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -29,6 +29,8 @@ export const EntrySchema = S.Struct({ seconds: S.Number, exitCode: S.Union([S.Int, S.Null]), shard: S.optional(ShardSchema), + ran: S.optional(S.Int), + reused: S.optional(S.Int), }) export type Entry = S.Schema.Type @@ -176,6 +178,19 @@ export const planJobs = ( return { jobs: [...shardJobs, ...wholeJobs] } } +const fullSetSecondsOf = (entry: Entry): number => + entry.ran === undefined || entry.reused === undefined || entry.ran === 0 + ? entry.seconds + : Math.round((entry.seconds * (entry.ran + entry.reused)) / entry.ran) + +const measuredOf = (entries: readonly Entry[]): number | undefined => { + const count = entries[0]?.shard?.count + if (count === undefined) return Math.max(...entries.map(fullSetSecondsOf)) + const indices = new Set(entries.map((entry) => entry.shard?.index)) + const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) + return complete ? entries.reduce((sum, entry) => sum + fullSetSecondsOf(entry), 0) : undefined +} + export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: string): TimingRecord => { const packages: Record = { ...previous.packages } const byPackage = new Map() @@ -183,14 +198,10 @@ export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: byPackage.set(entry.package, [...(byPackage.get(entry.package) ?? []), entry]) } for (const [name, entries] of byPackage) { - const count = entries[0]?.shard?.count - if (count === undefined) { - packages[name] = { seconds: Math.max(...entries.map((entry) => entry.seconds)), sha } - continue - } - const indices = new Set(entries.map((entry) => entry.shard?.index)) - const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) - if (complete) packages[name] = { seconds: entries.reduce((sum, entry) => sum + entry.seconds, 0), sha } + const evaluatedNothing = entries.some((entry) => entry.ran === 0) + const seconds = measuredOf(entries) + if (seconds === undefined || (evaluatedNothing && previous.packages[name] !== undefined)) continue + packages[name] = { seconds, sha } } return { version: 1, packages } } @@ -241,6 +252,7 @@ const RunFailedLineSchema = S.Struct({ const ReuseLineSchema = S.Struct({ _tag: S.Literal('reuse'), ran: S.Int.check(S.isGreaterThanOrEqualTo(0)), + reused: S.Int.check(S.isGreaterThanOrEqualTo(0)), }) const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder): ReadonlyArray => @@ -251,10 +263,10 @@ const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder< const terminalRecordOf = (streamText: string | null): Option.Option => Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record) -export type ReuseState = { readonly ran: number } +export type ReuseState = { readonly ran: number; readonly reused: number } const reuseOf = (streamText: string | null): Option.Option => - Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran })) + Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran, reused: line.reused })) export type ReportState = { reportText: string | null diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index eddb9bee6..fddd5716f 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -106,12 +106,18 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`) ok = false } + const seconds = Math.round((Date.now() - started) / 1000) + const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' + const reportsDir = join(dir, 'reports') + const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } + const state = await loadState(reportsDir, readText) if (exitCode !== null) { entries.push({ package: name, - seconds: Math.round((Date.now() - started) / 1000), + seconds, exitCode, ...(shard === undefined ? {} : { shard }), + ...Option.match(state.reuse, { onNone: () => ({}), onSome: ({ ran, reused }) => ({ ran, reused }) }), }) await Deno.mkdir('.timings', { recursive: true }) await Deno.writeTextFile( @@ -119,10 +125,6 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom JSON.stringify({ job: job.id, entries } satisfies Part), ) } - const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' - const reportsDir = join(dir, 'reports') - const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } - const state = await loadState(reportsDir, readText) console.log(buildSummary(input, state)) const missing = buildRequireError(input, state) if (missing !== null) { @@ -178,9 +180,12 @@ const preflightPackage = async (name: string, dir: string, capSeconds: number): const preflight = async (jobs: readonly Job[], capSeconds: number): Promise => { let ok = true + let coverage = false for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) { if (!await preflightPackage(name, dir, capSeconds)) ok = false + if ((await readIfPresent(join('preflight', dir, PREFLIGHT_FILE))) !== undefined) coverage = true } + await appendTo('GITHUB_OUTPUT', `coverage=${coverage}\n`) return ok }