diff --git a/.changeset/petite-insects-feel.md b/.changeset/petite-insects-feel.md new file mode 100644 index 000000000..44356c3ab --- /dev/null +++ b/.changeset/petite-insects-feel.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +A dry-run-only run with incremental mode on publishes its dry-run coverage to the incremental file, keeping any verdicts already there, so a later run that reads the file through incrementalSources skips its own dry run. The run-inputs digest no longer covers dryRunOnly, so a preflight and the runs that reuse its coverage share one digest; existing caches are refused once with runInputsChanged. diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 7129bdebb..43817f080 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -51,12 +51,93 @@ jobs: if-no-files-found: ignore retention-days: 1 - mutation: - name: mutation · ${{ matrix.job.name }} + preflight: + name: preflight · one dry run per package needs: plan if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} runs-on: ubuntu-latest + timeout-minutes: 45 + env: + OTEL_ENABLED: "true" + OTEL_SERVICE_NAME: stryker-js-mutation + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318 + OUT_DIR: mutation-telemetry + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: denoland/setup-deno@v2 + - name: Build + run: pnpm exec turbo run build --output-logs=errors-only --log-order=stream ${{ join(fromJSON(needs.plan.outputs.jobs).*.filters, ' ') }} + + - uses: actions/cache/restore@v6 + with: + path: .stryker-incremental + key: stryker-incremental-${{ github.run_id }} + restore-keys: | + stryker-incremental- + - name: Place incremental files + run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi + + - name: Install process-compose (pinned) + run: | + curl -sSfL -o /tmp/pc.tar.gz https://github.com/F1bonacc1/process-compose/releases/download/v1.122.0/process-compose_linux_amd64.tar.gz + echo '9b6dbc38324c0b0481f1cd1dd828ffdc78117129ec797678f4bf8c4023311281 /tmp/pc.tar.gz' | sha256sum -c - + sudo tar xzf /tmp/pc.tar.gz -C /usr/local/bin process-compose + - name: Start LGTM (blocks until probe-ready) + run: | + echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" + pnpm lgtm:up + + - name: Preflight + env: + JOBS: ${{ needs.plan.outputs.jobs }} + run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS" + + - uses: actions/upload-artifact@v7 + with: + name: mutation-preflight + path: preflight/ + if-no-files-found: ignore + retention-days: 1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: mutation-preflight-evidence + path: preflight-evidence/ + if-no-files-found: ignore + retention-days: 30 + + - name: Export telemetry + if: always() + continue-on-error: true + run: ./test/e2e/scripts/export-traces.ts + - name: Package telemetry + if: always() + run: if [ -d mutation-telemetry ]; then tar czf mutation-telemetry.tar.gz mutation-telemetry; fi + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mutation-telemetry-${{ github.run_id }}-preflight + path: mutation-telemetry.tar.gz + if-no-files-found: ignore + retention-days: 30 + + mutation: + name: mutation · ${{ matrix.job.name }} + needs: [plan, preflight] + if: ${{ needs.preflight.result == 'success' }} + runs-on: ubuntu-latest timeout-minutes: 75 + env: + OTEL_ENABLED: "true" + OTEL_SERVICE_NAME: stryker-js-mutation + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318 + OUT_DIR: mutation-telemetry strategy: fail-fast: false matrix: @@ -81,6 +162,21 @@ jobs: stryker-incremental- - name: Place incremental files run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi + - uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: mutation-preflight + path: . + + - name: Install process-compose (pinned) + run: | + curl -sSfL -o /tmp/pc.tar.gz https://github.com/F1bonacc1/process-compose/releases/download/v1.122.0/process-compose_linux_amd64.tar.gz + echo '9b6dbc38324c0b0481f1cd1dd828ffdc78117129ec797678f4bf8c4023311281 /tmp/pc.tar.gz' | sha256sum -c - + sudo tar xzf /tmp/pc.tar.gz -C /usr/local/bin process-compose + - name: Start LGTM (blocks until probe-ready) + run: | + echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" + pnpm lgtm:up - name: Mutation env: @@ -107,7 +203,22 @@ jobs: name: mutation-part-${{ matrix.job.id }} path: mutation-parts/ if-no-files-found: ignore - retention-days: 1 + retention-days: 30 + + - name: Export telemetry + if: always() + continue-on-error: true + run: ./test/e2e/scripts/export-traces.ts + - name: Package telemetry + if: always() + run: if [ -d mutation-telemetry ]; then tar czf mutation-telemetry.tar.gz mutation-telemetry; fi + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mutation-telemetry-${{ github.run_id }}-${{ matrix.job.id }} + path: mutation-telemetry.tar.gz + if-no-files-found: ignore + retention-days: 30 timings: name: record mutation timings @@ -138,13 +249,14 @@ jobs: report: name: Mutation report - needs: [plan, mutation] - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} + needs: [plan, preflight, mutation] + if: ${{ !cancelled() && needs.preflight.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 25 permissions: contents: read actions: write + security-events: write steps: - uses: actions/checkout@v7 - uses: pnpm/action-setup@v6 @@ -194,8 +306,31 @@ jobs: path: mutation-merged/* retention-days: 14 - - name: Delete per-job artifacts + - name: Merge SARIF if: always() + run: $MUTATION_JOB sarif --parts mutation-parts --out mutation-sarif/mutation.sarif + - uses: github/codeql-action/upload-sarif@v4 + if: ${{ always() && hashFiles('mutation-sarif/mutation.sarif') != '' }} + with: + sarif_file: mutation-sarif/mutation.sarif + category: mutation + + - name: Survivor baseline + if: ${{ env.MUTATION_REPORT_COMPLETE == 'true' && github.ref == 'refs/heads/main' }} + run: | + mkdir -p mutation-gate/reports/mutation + cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json + cd mutation-gate + "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json --update-baseline + - uses: actions/upload-artifact@v7 + if: ${{ env.MUTATION_REPORT_COMPLETE == 'true' && github.ref == 'refs/heads/main' }} + with: + name: mutation-survivor-baseline + path: mutation-gate/mutation-survivor-baseline.json + retention-days: 90 + + - name: Delete per-job artifacts + if: ${{ always() && needs.mutation.result == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ github.token }} diff --git a/packages/stryker-js/src/mutation-reporting.schema.ts b/packages/stryker-js/src/mutation-reporting.schema.ts index 229beec0c..a4ab614ab 100644 --- a/packages/stryker-js/src/mutation-reporting.schema.ts +++ b/packages/stryker-js/src/mutation-reporting.schema.ts @@ -10,3 +10,5 @@ export class ManifestUnreadable extends S.TaggedError()('Man } export const ManifestSchema = S.Struct({ version: S.optional(S.String) }) + +export const IncrementalReportObjectSchema = S.Record(S.String, S.Json) diff --git a/packages/stryker-js/src/mutation-reporting.service.ts b/packages/stryker-js/src/mutation-reporting.service.ts index a0d1180f3..c044aba0a 100644 --- a/packages/stryker-js/src/mutation-reporting.service.ts +++ b/packages/stryker-js/src/mutation-reporting.service.ts @@ -40,7 +40,7 @@ import { classifyExit, ClassifyExitCommand } from './classify-exit.workflow.js' import type { DryRunCoverage } from './dry-run-coverage.schema.js' import type { FormatIdentity, TimeoutEvidence, TimeoutKind } from './IncrementalDiff.schema.js' import { TimeoutEvidenceSchema } from './IncrementalDiff.schema.js' -import { ManifestSchema, ManifestUnreadable } from './mutation-reporting.schema.js' +import { IncrementalReportObjectSchema, ManifestSchema, ManifestUnreadable } from './mutation-reporting.schema.js' import type { ResolvedMode } from './output-mode.schema.js' import { ProjectFiles, type ProjectFilesShape } from './project-files.service.js' import type { Project, ProjectFile } from './Project.schema.js' @@ -135,6 +135,7 @@ export interface MutationReportingService { input: MutationReportingInput, plannedMutants: readonly Mutant.Mutant[], ) => Effect.Effect + readonly publishDryRunCoverage: (input: MutationReportingInput) => Effect.Effect } export class MutationReporting extends Context.Service()( @@ -158,6 +159,7 @@ export class MutationReporting extends Context.Service mapRunResult(mutant, result), reportAll: (input) => reportAll(deps, input), checkpoint: (input, plannedMutants) => checkpoint(deps, input, plannedMutants), + publishDryRunCoverage: (input) => publishDryRunCoverage(deps, input), }) }), ) @@ -928,6 +930,40 @@ const checkpoint = ( onFalse: () => Effect.void, }) +const priorIncrementalObjectOf = (text: string): Option.Option => + S.decodeOption(S.fromJsonString(IncrementalReportObjectSchema))(text) + +const dryRunCoverageReportOf = Effect.fnUntraced(function*( + deps: MutationReportingDeps, + input: MutationReportingInput, + coverage: DryRunCoverage, +) { + const prior = yield* deps.fs.readFileString(input.options.incrementalFile).pipe(Effect.option) + return yield* Option.match(Option.flatMap(prior, priorIncrementalObjectOf), { + onSome: (report) => Effect.succeed({ ...report, dryRunCoverage: coverage }), + onNone: () => slimIncrementalReport(deps, input, []), + }) +}) + +const writeDryRunCoverage = Effect.fnUntraced(function*( + deps: MutationReportingDeps, + input: MutationReportingInput, + coverage: DryRunCoverage, +) { + const report = yield* dryRunCoverageReportOf(deps, input, coverage) + const json = yield* S.encodeEffect(S.fromJsonString(S.Unknown))(report).pipe(Effect.orDie) + yield* writeAtomic(deps, input.options.incrementalFile, json) +}) + +const publishDryRunCoverage = (deps: MutationReportingDeps, input: MutationReportingInput) => + Option.match( + Option.filter(Option.fromUndefinedOr(input.testCoverage.dryRunCoverage), () => input.options.incremental), + { + onNone: () => Effect.void, + onSome: (coverage) => writeDryRunCoverage(deps, input, coverage), + }, + ) + if (import.meta.vitest !== void 0) { const { it } = await import('@systemfsoftware/vitest') const { Mutant: { Mutant, MutantStatusSchema }, TestRunner: { MutantRunResultSchema } } = await import( diff --git a/packages/stryker-js/src/run/mutation-test.cell.ts b/packages/stryker-js/src/run/mutation-test.cell.ts index ceb531652..6a8eea30b 100644 --- a/packages/stryker-js/src/run/mutation-test.cell.ts +++ b/packages/stryker-js/src/run/mutation-test.cell.ts @@ -70,7 +70,15 @@ const writeMutationTestNoTests = Effect.fn(SpanTaxonomy.Spans.mutationTestNoTest return { results: [], verdict: null } }) -const writeMutationTestDryRunOnly = Effect.fn(SpanTaxonomy.Spans.mutationTestDryRunOnly.name)(function*() { +const writeMutationTestDryRunOnly = Effect.fn(SpanTaxonomy.Spans.mutationTestDryRunOnly.name)(function*( + raw: MutationTestRaw, +) { + const reporting = yield* MutationReporting + const env = yield* RunEnvironment + yield* reporting.publishDryRunCoverage(reportingInputOf({ prev: raw.prev, env, results: [] })).pipe( + Effect.tapCause((cause) => Effect.logWarning('Failed to publish the dry-run coverage', cause)), + Effect.ignoreCause, + ) yield* phaseEntered('mutation-test') yield* Effect.logInfo('The dry-run has been completed successfully. No mutations have been executed.') return { results: [], verdict: null } @@ -272,7 +280,8 @@ export const mutationTestCell = Sandwich.named( return raw }) ).decide(admitMutationTest).write({ - MutationTestDryRunOnly: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestDryRunOnly() }), + MutationTestDryRunOnly: (_decision, raw) => + writeMutationTestOutcome({ raw, outcome: writeMutationTestDryRunOnly(raw) }), MutationTestNoTests: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestNoTests() }), MutationTestProceed: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestProceed(raw) }), RunFailure: (failure) => Effect.fail(RunFailure.make(failure)), diff --git a/packages/stryker-js/src/verdict-semantics.ts b/packages/stryker-js/src/verdict-semantics.ts index 08e980b21..31ddefa0d 100644 --- a/packages/stryker-js/src/verdict-semantics.ts +++ b/packages/stryker-js/src/verdict-semantics.ts @@ -72,7 +72,7 @@ const withoutVersionOf = (value: Json): Json => onNone: () => value, }) -const scopeOptionKeys: readonly string[] = ['mutate', 'since', 'mutantIds'] +const scopeOptionKeys: readonly string[] = ['mutate', 'since', 'mutantIds', 'dryRunOnly'] const presentationOptionKeys: readonly string[] = [ 'reporters', diff --git a/packages/stryker-js/tests/incremental-reuse.integration.test.ts b/packages/stryker-js/tests/incremental-reuse.integration.test.ts index c8cbf5841..e628d8d31 100644 --- a/packages/stryker-js/tests/incremental-reuse.integration.test.ts +++ b/packages/stryker-js/tests/incremental-reuse.integration.test.ts @@ -440,6 +440,106 @@ Feature('Content-keyed reuse across incremental reports') ), ) + scenario( + 'A dry-run-only preflight publishes its coverage, and a run reading it through incrementalSources skips its own dry run', + Gherkin.Do.pipe( + Given('a workspace whose command runner appends every spawn to a log file')( + 'fixture', + () => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const root = yield* writeFixture([['src/math.ts', SOURCE]]) + const spawnLog = yield* fs.makeTempFile({ prefix: 'runner-spawns', suffix: '.txt' }) + const commandRunner = { command: `echo spawned >> ${spawnLog}` } + return yield* Effect.ensuring( + Effect.gen(function*() { + const cold = yield* runOnce( + root, + optionsOf(root, { commandRunner, incrementalFile: `${root}/reports/cold.json` }), + ) + const spawnsInColdRun = yield* lineCountOf(spawnLog) + yield* runOnce( + root, + optionsOf(root, { + commandRunner, + dryRunOnly: true, + incrementalFile: `${root}/reports/preflight.json`, + }), + ) + const spawnsAfterPreflight = yield* lineCountOf(spawnLog) + const preflighted = yield* runOnce( + root, + optionsOf(root, { commandRunner, incrementalSources: ['reports/preflight.json'] }), + ) + const spawnsAfterPreflighted = yield* lineCountOf(spawnLog) + return { + cold, + preflighted, + spawnsInColdRun, + spawnsInPreflight: spawnsAfterPreflight - spawnsInColdRun, + spawnsInPreflightedRun: spawnsAfterPreflighted - spawnsAfterPreflight, + } + }), + Effect.andThen(removeFixture(root), Effect.orDie(fs.remove(spawnLog, { force: true }))), + ) + }).pipe(Effect.orDie, Effect.provide(filePorts)), + ), + Then( + 'the preflight spawns exactly the dry run a cold run spawns, and the preflighted run spawns only its mutants', + )((s, expect) => { + const coldDryRunSpawns = s.fixture.spawnsInColdRun - (s.fixture.cold.reuse?.ran ?? 0) + return expect({ + coldRanMutants: (s.fixture.cold.reuse?.ran ?? 0) > 0, + coldDryRunSpawned: coldDryRunSpawns > 0, + preflightSpawns: s.fixture.spawnsInPreflight, + preflightedRan: s.fixture.preflighted.reuse?.ran, + preflightedDryRunSpawns: s.fixture.spawnsInPreflightedRun - (s.fixture.preflighted.reuse?.ran ?? 0), + }).toEqual({ + coldRanMutants: true, + coldDryRunSpawned: true, + preflightSpawns: coldDryRunSpawns, + preflightedRan: s.fixture.cold.reuse?.ran, + preflightedDryRunSpawns: 0, + }) + }), + ), + ) + + scenario( + 'A dry-run-only run into an existing incremental file keeps the verdicts it holds', + Gherkin.Do.pipe( + Given('a workspace whose incremental file already holds every verdict')( + 'fixture', + () => + Effect.gen(function*() { + const root = yield* writeFixture([['src/math.ts', SOURCE]]) + return yield* Effect.ensuring( + Effect.gen(function*() { + const first = yield* runOnce(root, optionsOf(root)) + yield* runOnce(root, optionsOf(root, { dryRunOnly: true })) + const after = yield* runOnce(root, optionsOf(root)) + return { first, after } + }), + removeFixture(root), + ) + }).pipe(Effect.orDie, Effect.provide(filePorts)), + ), + Then('the run after the dry-run-only run reuses every verdict the first run wrote')((s, expect) => + expect({ + plannedNonZero: s.fixture.first.mutants.length > 0, + after: { + reused: s.fixture.after.reuse?.reused, + ran: s.fixture.after.reuse?.ran, + refused: s.fixture.after.reuse?.refused, + }, + }).toEqual({ + plannedNonZero: true, + after: { reused: s.fixture.first.mutants.length, ran: 0, refused: ZERO_REFUSALS }, + }) + ), + ), + ) + scenario( 'A verdict written under another report path is reused through the incrementalSources globs', Gherkin.Do.pipe( diff --git a/packages/toolchain/stryker-config/lib/base.js b/packages/toolchain/stryker-config/lib/base.js index bc7b3d212..da8d5ccec 100644 --- a/packages/toolchain/stryker-config/lib/base.js +++ b/packages/toolchain/stryker-config/lib/base.js @@ -13,7 +13,7 @@ const restoredShardReports = ['reports/stryker-incremental-*.json'] export const sharedConfig = { packageManager: 'pnpm', - reporters: isAgent || isCI ? ['json', 'html'] : ['progress', 'html', 'json'], + reporters: isAgent ? ['json', 'html'] : isCI ? ['json', 'html', 'sarif'] : ['progress', 'html', 'json'], htmlReporter: { fileName: 'reports/mutation-report.html' }, jsonReporter: { fileName: 'reports/mutation-report.json' }, coverageAnalysis: 'perTest', diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index 40732c9a4..ed3d3abc8 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -1,21 +1,29 @@ import { assertEquals, assertStringIncludes, assertThrows } from '@std/assert' +import * as Option from 'effect/Option' import { + buildPreflightError, + buildPreflightSummary, buildRequireError, buildSummary, combineParts, decodeInput, decodeJson, emptyRecord, + evaluatedOf, + evaluatedSummaryOf, JobOrNullSchema, JobsSchema, loadState, mergeRecord, + mergeSarif, type Part, PartMetaSchema, planJobs, PnpmWorkspaceSchema, + PREFLIGHT_FILE, ReportSchema, + SarifLogSchema, type StagedPart, type SummaryInput, type TimingRecord, @@ -240,3 +248,84 @@ Deno.test('decodeInput names the workspace file when packages is malformed', () 'pnpm-workspace.yaml', ) }) + +Deno.test('a passing preflight names where its coverage went, or that every shard repeats the dry run', async () => { + const state = await loadState(dir, readFileFor({})) + const passed = inputOf({ outcome: 'success', exitCode: 0 }) + assertStringIncludes(buildPreflightSummary(passed, state, true), `packages/x/${PREFLIGHT_FILE}`) + assertStringIncludes(buildPreflightSummary(passed, state, false), 'every shard runs its own dry run') + assertEquals(buildPreflightSummary(passed, state, true).includes('RecordMissing'), false) + assertEquals(buildPreflightError(passed, state), null) +}) + +Deno.test('a failing preflight renders its terminal record and annotates it', async () => { + const state = await loadState(dir, readFileFor({ [`${dir}/mutation-stream.jsonl`]: failureStream })) + const failed = inputOf({ exitCode: 5 }) + assertStringIncludes(buildPreflightSummary(failed, state, false), 'BaselineTestsFailed') + assertStringIncludes(buildPreflightError(failed, state) ?? '', 'BaselineTestsFailed') +}) + +const reuseLine = (ran: number): string => `${JSON.stringify({ _tag: 'reuse', reused: 4, ran, refused: {} })}\n` + +Deno.test('evaluatedOf sums the mutants every part ran and reports evaluated-none when that sum is zero', () => { + assertEquals(evaluatedOf([reuseLine(2), reuseLine(3), '{"_tag":"tick"}\n']), Option.some(5)) + assertStringIncludes(evaluatedSummaryOf(evaluatedOf([reuseLine(0), reuseLine(0)])), 'no mutants') + assertEquals(evaluatedOf(['{"_tag":"tick"}\n']), Option.none()) +}) + +const sarifOf = ( + rules: readonly string[], + results: readonly (readonly [string, string])[], + invocations?: unknown[], +) => ({ + $schema: 'https://json.schemastore.org/sarif-2.1.0.json', + version: '2.1.0' as const, + runs: [{ + tool: { + driver: { + name: 'StrykerJS', + version: '1', + informationUri: 'https://stryker-mutator.io', + rules: rules.map((id) => ({ id })), + }, + }, + results: results.map(([ruleId, uri]) => ({ + ruleId, + ruleIndex: rules.indexOf(ruleId), + locations: [{ physicalLocation: { artifactLocation: { uri }, region: { startLine: 1 } } }], + })), + ...(invocations === undefined ? {} : { invocations }), + }], +}) + +Deno.test('mergeSarif folds every package into one run with repository paths and re-indexed rules', () => { + const notification = { + descriptor: { id: 'BaselineTestsFailed' }, + locations: [{ physicalLocation: { artifactLocation: { uri: 'src/b.test.ts' }, region: { startLine: 3 } } }], + } + const merged = Option.getOrThrow(mergeSarif([ + { dir: 'packages/a', log: decodeInput(SarifLogSchema, sarifOf(['Z'], [['Z', 'src/a.ts']]), 'a') }, + { + dir: 'packages/b', + log: decodeInput( + SarifLogSchema, + sarifOf(['A', 'Z'], [['A', 'src/b.ts']], [{ toolExecutionNotifications: [notification] }]), + 'b', + ), + }, + ])) + const [run] = merged.runs + assertEquals(merged.runs.length, 1) + assertEquals(run?.tool.driver.rules.map((rule) => rule.id), ['A', 'Z']) + assertEquals( + run?.results.map(( + result, + ) => [result.ruleId, result['ruleIndex'], result.locations[0]?.physicalLocation.artifactLocation.uri]), + [['Z', 1, 'packages/a/src/a.ts'], ['A', 0, 'packages/b/src/b.ts']], + ) + assertEquals( + run?.invocations?.[0]?.toolExecutionNotifications?.[0]?.locations?.[0]?.physicalLocation.artifactLocation.uri, + 'packages/b/src/b.test.ts', + ) + assertEquals(mergeSarif([]), Option.none()) +}) diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index a9ab490d0..24fe1ff34 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -364,6 +364,123 @@ export const buildRequireError = (input: SummaryInput, state: ReportState): stri return annotationsOf(recordFor(input, state)).join('\n') } +export const PREFLIGHT_FILE = 'reports/stryker-incremental-preflight.json' + +export const buildPreflightSummary = (input: SummaryInput, state: ReportState, published: boolean): string => { + const header = [`#### Mutation preflight · **${input.package}**`, ''] + if (input.outcome === 'failure') { + return `${[...header, '- **Dry run**: **failure**', markdownOf(recordFor(input, state))].join('\n')}\n` + } + const coverage = published + ? `- **Dry-run coverage**: published at **${input.package}/${PREFLIGHT_FILE}**; shards whose digests match reuse it` + : '- **Dry-run coverage**: not published by this CLI; every shard runs its own dry run' + return `${[...header, '- **Dry run**: **passed**', coverage].join('\n')}\n` +} + +export const buildPreflightError = (input: SummaryInput, state: ReportState): string | null => + input.outcome === 'success' ? null : annotationsOf(recordFor(input, state)).join('\n') + +export const evaluatedOf = (streams: readonly string[]): Option.Option => { + const reuses = streams.flatMap((stream) => Option.toArray(reuseOf(stream))) + return reuses.length === 0 ? Option.none() : Option.some(reuses.reduce((sum, reuse) => sum + reuse.ran, 0)) +} + +export const evaluatedSummaryOf = (evaluated: Option.Option): string => + Option.match(evaluated, { + onNone: () => '- **Evaluated**: no part reported which mutants it ran', + onSome: (ran) => + ran === 0 + ? '- **Evaluated**: **no mutants**; every verdict was reused, so this run tested nothing' + : `- **Evaluated**: **${ran}** mutant(s); every other verdict was reused`, + }) + +const SarifRest = [S.Record(S.String, S.Json)] as const +const SarifArtifactLocationSchema = S.StructWithRest(S.Struct({ uri: S.String }), SarifRest) +const SarifPhysicalLocationSchema = S.StructWithRest( + S.Struct({ artifactLocation: SarifArtifactLocationSchema }), + SarifRest, +) +const SarifLocationSchema = S.StructWithRest(S.Struct({ physicalLocation: SarifPhysicalLocationSchema }), SarifRest) +const SarifRuleSchema = S.StructWithRest(S.Struct({ id: S.String }), SarifRest) +const SarifResultSchema = S.StructWithRest( + S.Struct({ ruleId: S.String, locations: S.Array(SarifLocationSchema) }), + SarifRest, +) +const SarifNotificationSchema = S.StructWithRest( + S.Struct({ locations: S.optionalKey(S.Array(SarifLocationSchema)) }), + SarifRest, +) +const SarifInvocationSchema = S.StructWithRest( + S.Struct({ toolExecutionNotifications: S.optionalKey(S.Array(SarifNotificationSchema)) }), + SarifRest, +) +const SarifDriverSchema = S.StructWithRest(S.Struct({ name: S.String, rules: S.Array(SarifRuleSchema) }), SarifRest) +const SarifRunSchema = S.Struct({ + tool: S.Struct({ driver: SarifDriverSchema }), + results: S.Array(SarifResultSchema), + invocations: S.optionalKey(S.Array(SarifInvocationSchema)), +}) +export const SarifLogSchema = S.Struct({ + $schema: S.String, + version: S.Literal('2.1.0'), + runs: S.Array(SarifRunSchema), +}) +export type SarifLog = S.Schema.Type +type SarifLocation = S.Schema.Type +type SarifRun = S.Schema.Type + +export type PackageSarif = { readonly dir: string; readonly log: SarifLog } + +const underDir = (dir: string) => (location: SarifLocation): SarifLocation => ({ + ...location, + physicalLocation: { + ...location.physicalLocation, + artifactLocation: { + ...location.physicalLocation.artifactLocation, + uri: `${dir}/${location.physicalLocation.artifactLocation.uri}`, + }, + }, +}) + +const invocationsUnder = (dir: string, run: SarifRun) => + (run.invocations ?? []).map((invocation) => ({ + ...invocation, + ...(invocation.toolExecutionNotifications === undefined ? {} : { + toolExecutionNotifications: invocation.toolExecutionNotifications.map((notification) => ({ + ...notification, + ...(notification.locations === undefined ? {} : { locations: notification.locations.map(underDir(dir)) }), + })), + }), + })) + +export const mergeSarif = (packages: readonly PackageSarif[]): Option.Option => { + const runs = packages.flatMap(({ dir, log }) => log.runs.map((run) => ({ dir, run }))) + return Option.map(Arr.head(runs), ({ run: first }) => { + const rules = Arr.dedupeWith( + runs.flatMap(({ run }) => run.tool.driver.rules).sort((a, b) => a.id.localeCompare(b.id)), + (a, b) => a.id === b.id, + ) + const ruleIndex = new Map(rules.map((rule, index) => [rule.id, index] as const)) + const results = runs.flatMap(({ dir, run }) => + run.results.map((result) => ({ + ...result, + ruleIndex: ruleIndex.get(result.ruleId) ?? 0, + locations: result.locations.map(underDir(dir)), + })) + ) + const invocations = runs.flatMap(({ dir, run }) => invocationsUnder(dir, run)) + return { + $schema: packages[0]?.log.$schema ?? '', + version: '2.1.0', + runs: [{ + tool: { driver: { ...first.tool.driver, rules } }, + results, + ...(invocations.length === 0 ? {} : { invocations }), + }], + } + }) +} + export const combineParts = (parts: readonly StagedPart[]): Map => { const byPackage = Map.groupBy(parts, (part) => part.meta.package) const combined = new Map() diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index 4114dfcc2..eddb9bee6 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -3,28 +3,39 @@ import { parseArgs } from '@std/cli/parse-args' import { expandGlob } from '@std/fs/expand-glob' import { dirname, join } from '@std/path' +import * as Option from 'effect/Option' import { + buildPreflightError, + buildPreflightSummary, buildRequireError, buildSummary, type CombinedPart, combineParts, decodeJson, type Entry, + evaluatedOf, + evaluatedSummaryOf, type Job, JobOrNullSchema, JobsSchema, loadState, + mergeSarif, type Outcome, + type PackageSarif, type Part, type PartMeta, PartMetaSchema, + PREFLIGHT_FILE, ReportSchema, + SarifLogSchema, type Shard, slugOf, type StagedPart, } from './lib/mutation-plan.ts' +const PART_SARIF = 'mutation.sarif' + const incrementalFileOf = (shard: Shard | undefined): string => shard === undefined ? 'reports/stryker-incremental.json' @@ -59,18 +70,14 @@ const removeOutputsOfEarlierRun = async (reportsDir: string): Promise => { } } -const strykerUnderCap = async (name: string, shard: Shard | undefined, capSeconds: number): Promise => { +const strykerUnderCap = async ( + name: string, + shard: Shard | undefined, + capSeconds: number, + args: readonly string[], +): Promise => { const { code } = await new Deno.Command('timeout', { - args: [ - '--kill-after=60', - String(capSeconds), - 'pnpm', - '--filter', - name, - 'mutation', - '--incrementalFile', - incrementalFileOf(shard), - ], + args: ['--kill-after=60', String(capSeconds), 'pnpm', '--filter', name, 'mutation', ...args], env: { STRYKER_SHARD: shard === undefined ? '' : `${shard.index}/${shard.count}` }, stdout: 'inherit', stderr: 'inherit', @@ -93,7 +100,7 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom await removeOutputsOfEarlierRun(join(dir, 'reports')) if (cap > 0) { console.log(`::group::mutation ${labelOf(dir, shard)}`) - exitCode = await strykerUnderCap(name, shard, cap) + exitCode = await strykerUnderCap(name, shard, cap, ['--incrementalFile', incrementalFileOf(shard)]) console.log('::endgroup::') } else { console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`) @@ -131,6 +138,8 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom ) await copyIfPresent(join(reportsDir, 'mutation-report.json'), join(part, 'mutation-report.json')) await copyIfPresent(join(reportsDir, 'mutation-stream.jsonl'), join(part, 'mutation-stream.jsonl')) + await copyIfPresent(join(reportsDir, 'mutation-report.sarif'), join(part, PART_SARIF)) + await copyIfPresent(join(reportsDir, 'mutation', 'failure.json'), join(part, 'failure.json')) await copyIfPresent(join(dir, incrementalFile), join('incremental', dir, incrementalFile)) } return ok @@ -138,6 +147,60 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom const plannedPackages = (jobs: readonly Job[]): string[] => [...new Set(jobs.flatMap((job) => job.dirs))].sort() +const namedPackagesOf = (jobs: readonly Job[]): ReadonlyMap => + new Map(jobs.flatMap((job) => job.dirs.map((dir, position) => [dir, job.packages[position] ?? dir] as const))) + +const preflightPackage = async (name: string, dir: string, capSeconds: number): Promise => { + const reportsDir = join(dir, 'reports') + await removeOutputsOfEarlierRun(reportsDir) + console.log(`::group::preflight ${dir}`) + const exitCode = await strykerUnderCap(name, undefined, capSeconds, [ + '--dryRunOnly', + '--incrementalFile', + PREFLIGHT_FILE, + ]) + console.log('::endgroup::') + const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' + const input = { package: dir, outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: capSeconds } + const state = await loadState(reportsDir, readText) + const published = (await readIfPresent(join(dir, PREFLIGHT_FILE))) !== undefined + console.log(buildPreflightSummary(input, state, published)) + const error = buildPreflightError(input, state) + if (error !== null) console.log(error) + await copyIfPresent(join(dir, PREFLIGHT_FILE), join('preflight', dir, PREFLIGHT_FILE)) + await copyIfPresent( + join(reportsDir, 'mutation-stream.jsonl'), + join('preflight-evidence', dir, 'mutation-stream.jsonl'), + ) + await copyIfPresent(join(reportsDir, 'mutation', 'failure.json'), join('preflight-evidence', dir, 'failure.json')) + return outcome === 'success' +} + +const preflight = async (jobs: readonly Job[], capSeconds: number): Promise => { + let ok = true + for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) { + if (!await preflightPackage(name, dir, capSeconds)) ok = false + } + return ok +} + +const readPackageSarifs = async (root: string): Promise => { + const sarifs: PackageSarif[] = [] + for await (const marker of expandGlob('**/mutation-part.json', { root })) { + const sarifPath = join(dirname(marker.path), PART_SARIF) + const text = await readIfPresent(sarifPath) + if (text === undefined) continue + const meta = decodeJson(PartMetaSchema, await Deno.readTextFile(marker.path), marker.path) + sarifs.push({ dir: meta.package, log: decodeJson(SarifLogSchema, text, sarifPath) }) + } + return sarifs +} + +const appendTo = async (variable: string, text: string): Promise => { + const file = Deno.env.get(variable) + if (file !== undefined && file !== '') await Deno.writeTextFile(file, text, { append: true }) +} + const readStagedParts = async (root: string): Promise => { const parts: StagedPart[] = [] for await (const marker of expandGlob('**/mutation-part.json', { root })) { @@ -178,19 +241,42 @@ const main = async (): Promise => { if (!await runJob(job, Number(args['cap-seconds']), Number(args['budget-seconds']))) Deno.exit(1) return } + if (command === 'preflight') { + if (args['cap-seconds'] === undefined) throw new Error('preflight needs --cap-seconds') + const jobs = decodeJson(JobsSchema, Deno.env.get('JOBS') ?? '[]', 'JOBS') + if (!await preflight(jobs, Number(args['cap-seconds']))) Deno.exit(1) + return + } if (command === 'combine') { if (args.parts === undefined || args.out === undefined) throw new Error('combine needs --parts and --out') const jobs = decodeJson(JobsSchema, Deno.env.get('JOBS') ?? '[]', 'JOBS') - await writeCombined(args.out, combineParts(await readStagedParts(args.parts))) - const envFile = Deno.env.get('GITHUB_ENV') + const staged = await readStagedParts(args.parts) + const combined = combineParts(staged) + await writeCombined(args.out, combined) const packages = JSON.stringify(plannedPackages(jobs)) - if (envFile !== undefined && envFile !== '') { - await Deno.writeTextFile(envFile, `PACKAGES=${packages}\n`, { append: true }) + const complete = combined.size > 0 && [...combined.values()].every((part) => part.report !== undefined) + await appendTo('GITHUB_ENV', `PACKAGES=${packages}\nMUTATION_REPORT_COMPLETE=${complete}\n`) + const evaluated = evaluatedOf(staged.flatMap((part) => (part.stream === undefined ? [] : [part.stream]))) + const evaluatedLine = evaluatedSummaryOf(evaluated) + await appendTo('GITHUB_STEP_SUMMARY', `${evaluatedLine}\n`) + if (Option.contains(evaluated, 0)) { + console.log('::notice title=Mutation evaluated no mutants::every verdict was reused, so this run tested nothing') } console.log(`planned packages: ${packages}`) + console.log(evaluatedLine) + return + } + if (command === 'sarif') { + if (args.parts === undefined || args.out === undefined) throw new Error('sarif needs --parts and --out') + const merged = Option.getOrUndefined(mergeSarif(await readPackageSarifs(args.parts))) + if (merged !== undefined) { + await Deno.mkdir(dirname(args.out), { recursive: true }) + await Deno.writeTextFile(args.out, JSON.stringify(merged)) + } + console.log(merged === undefined ? 'no part carried a SARIF log' : `wrote ${args.out}`) return } - throw new Error(`unknown command ${command ?? '(none)'}: expected run or combine`) + throw new Error(`unknown command ${command ?? '(none)'}: expected run, preflight, combine or sarif`) } if (import.meta.main) await main()