From 931594ea99cba1b9b776a2d10eddf355c9be083d Mon Sep 17 00:00:00 2001 From: Roomote Date: Fri, 11 Sep 2026 23:44:53 +0000 Subject: [PATCH 1/2] ci: make mutation findings advisory --- .github/workflows/mutation-testing.yml | 11 +-- scripts/stryker-diff.mjs | 63 +++++++++------- scripts/stryker-diff.test.mjs | 99 +++++++++++++++++--------- 3 files changed, 109 insertions(+), 64 deletions(-) diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index 620f119440..f6184b8154 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -22,8 +22,10 @@ jobs: - name: Record merge-queue enforcement if: github.event_name == 'merge_group' run: | - echo "## Changed-code mutation testing" >> "$GITHUB_STEP_SUMMARY" - echo "Mutation testing was enforced on each pull request before it entered the merge queue." >> "$GITHUB_STEP_SUMMARY" + { + echo "## Changed-code mutation testing" + echo "Mutation testing was enforced on each pull request before it entered the merge queue." + } >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary" - name: Checkout pull request merge result if: github.event_name == 'pull_request' @@ -50,7 +52,7 @@ jobs: if: github.event_name == 'pull_request' run: pnpm test:mutation-ci - - name: Mutate changed executable lines + - name: Enforce executable-line scope and run advisory mutation testing if: github.event_name == 'pull_request' env: BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -60,6 +62,7 @@ jobs: - name: Upload mutation reports id: mutation_report if: always() && github.event_name == 'pull_request' + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: changed-code-mutation-report @@ -76,4 +79,4 @@ jobs: echo "" echo "### Download mutation reports" echo "[Open the changed-code-mutation-report artifact]($ARTIFACT_URL), then open the package's mutation.html file." - } >> "$GITHUB_STEP_SUMMARY" + } >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary" diff --git a/scripts/stryker-diff.mjs b/scripts/stryker-diff.mjs index 2d6adddfa2..3fb7a34093 100644 --- a/scripts/stryker-diff.mjs +++ b/scripts/stryker-diff.mjs @@ -198,6 +198,7 @@ export function packageForPath(filePath) { export function buildManifest(entries, readSource, diffForPath) { const packages = new Map() + const advisories = [] for (const entry of entries) { if (!new Set(["A", "M", "R"]).has(entry.status)) continue @@ -211,7 +212,11 @@ export function buildManifest(entries, readSource, diffForPath) { ? new Set(Array.from({ length: sourceLineCount }, (_, index) => index + 1)) : parseChangedLines(diffForPath(entry.path)) - validateDisableDirectives(source, new Set(source.split(/\r?\n/).map((_, index) => index + 1)), entry.path) + try { + validateDisableDirectives(source, new Set(source.split(/\r?\n/).map((_, index) => index + 1)), entry.path) + } catch (error) { + advisories.push(error.message) + } const executableLines = executableChangedLines(source, changedLines, entry.path) if (executableLines.size === 0) continue @@ -243,7 +248,7 @@ export function buildManifest(entries, readSource, diffForPath) { } } - return { packages: [...packages.values()] } + return { packages: [...packages.values()], advisories } } function validateSha(value, name) { @@ -446,7 +451,7 @@ function escapeWorkflowProperty(value) { } export function formatAnnotationCommand(annotation) { - return `::error file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test gap::${escapeWorkflowData(annotation.message)}` + return `::warning file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test advisory::${escapeWorkflowData(annotation.message)}` } export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) { @@ -523,7 +528,7 @@ export function formatBlockingMutants(blockingMutants, packageRoot) { return lines } -export function formatSummary(rows, failures, manifest = {}) { +export function formatSummary(rows, advisories, manifest = {}) { const lines = [ "## Changed-code mutation testing", "", @@ -557,7 +562,7 @@ export function formatSummary(rows, failures, manifest = {}) { "", "### All surviving and uncovered mutants", "", - "Annotations highlight up to 20 unique locations (maximum 7 per file). This summary lists every blocking mutant.", + "Warning annotations highlight up to 20 unique locations (maximum 7 per file). This summary lists every advisory mutant.", "", ) for (const row of blockingRows) { @@ -600,14 +605,14 @@ export function formatSummary(rows, failures, manifest = {}) { ) } - if (failures.length > 0) { + if (advisories.length > 0) { lines.push( "", - "### Failures", + "### Advisory findings", "", - ...failures.map((failure) => { + ...advisories.map((advisory) => { const detail = - failure.length > 4_000 ? `${failure.slice(0, 4_000)}\n[truncated; see the step log]` : failure + advisory.length > 4_000 ? `${advisory.slice(0, 4_000)}\n[truncated; see the step log]` : advisory return `- ${detail.replaceAll("\n", "\n ")}` }), ) @@ -616,37 +621,44 @@ export function formatSummary(rows, failures, manifest = {}) { return `${lines.join("\n")}\n` } -function appendSummary(rows, failures, manifest) { +export function appendSummary(rows, advisories, manifest) { if (!process.env.GITHUB_STEP_SUMMARY) return - fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, failures, manifest)) + try { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, advisories, manifest)) + } catch (error) { + console.warn( + `::warning title=Mutation test advisory::Could not write the job summary: ${escapeWorkflowData(error.message)}`, + ) + } } export function evaluateReport(report, packageEntry) { const counts = mutantCounts(report) + const advisories = [] if (counts.valid > MAX_MUTANTS) { - throw new Error( + advisories.push( `${packageEntry.id} generated ${counts.valid} valid mutants (limit ${MAX_MUTANTS}). ` + "Split the PR or obtain a maintainer-reviewed narrow exclusion.", ) } if (counts.timeout > 10 || (counts.valid > 0 && counts.timeout / counts.valid > 0.15)) { - throw new Error( + advisories.push( `${packageEntry.id} timed out ${counts.timeout} of ${counts.valid} valid mutants. ` + - "The result is inconclusive; fix flaky or slow tests, or reduce the changed scope before merge.", + "The result is inconclusive; consider fixing flaky or slow tests, or reducing the changed scope.", ) } if (counts.blocking.length > 0) { - throw new Error( + advisories.push( `${packageEntry.id} has ${counts.survived} surviving and ${counts.noCoverage} uncovered changed-code mutants. ` + - "Add or strengthen focused tests before merge.", + "Consider adding or strengthening focused tests.", ) } - return counts + return { ...counts, advisories } } export function runManifest(repoRoot, manifest, reportRoot) { const rows = [] - const failures = [] + const advisories = [...(manifest.advisories ?? [])] const annotationState = { total: 0, perFile: new Map() } for (const packageEntry of manifest.packages) { @@ -694,7 +706,7 @@ export function runManifest(repoRoot, manifest, reportRoot) { const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json") const report = JSON.parse(fs.readFileSync(jsonReportPath, "utf8")) packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles) - counts = mutantCounts(report) + counts = evaluateReport(report, packageEntry) for (const annotation of formatAnnotations( counts.blocking, packageEntry.runRoot ?? packageEntry.root, @@ -702,7 +714,7 @@ export function runManifest(repoRoot, manifest, reportRoot) { )) { console.log(formatAnnotationCommand(annotation)) } - evaluateReport(report, packageEntry) + advisories.push(...counts.advisories) rows.push({ id: packageEntry.id, root: packageEntry.root, @@ -712,10 +724,10 @@ export function runManifest(repoRoot, manifest, reportRoot) { reportPath, changedLines: packageEntry.changedExecutableLines, ...counts, - result: "Passed", + result: counts.advisories.length > 0 ? "Advisory findings" : "Passed", }) } catch (error) { - failures.push(error.message) + advisories.push(error.message) rows.push({ id: packageEntry.id, root: packageEntry.root, @@ -730,13 +742,12 @@ export function runManifest(repoRoot, manifest, reportRoot) { survived: counts?.survived ?? 0, noCoverage: counts?.noCoverage ?? 0, blocking: counts?.blocking ?? [], - result: "Failed", + result: "Advisory incomplete", }) } } - appendSummary(rows, failures, manifest) - if (failures.length > 0) throw new Error(failures.join("\n")) + appendSummary(rows, advisories, manifest) return rows } @@ -758,7 +769,7 @@ function main() { const reportRoot = path.resolve(repoRoot, argument("--reports") ?? "reports/mutation") const manifest = selectFromGit(repoRoot, baseSha, headSha) if (manifest.packages.length === 0) { - appendSummary([], [], manifest) + appendSummary([], manifest.advisories, manifest) console.log("No changed executable lines in mutation-tested packages; mutation testing is not applicable.") return } diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index 931840606f..0e8d280667 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -10,6 +10,7 @@ import { MAX_CHANGED_LINES, MAX_MUTANTS, PACKAGE_CONFIGS, + appendSummary, buildManifest, discoverRelatedTestFiles, evaluateReport, @@ -53,6 +54,11 @@ describe("mutation testing workflow", () => { assert.ok(!workflow.includes("HEAD_SHA: ${{ github.event.pull_request.head.sha }}")) assert.ok(workflow.includes("steps.mutation_report.outputs.artifact-url")) assert.ok(workflow.includes("open the package's mutation.html file")) + assert.ok(workflow.includes("Enforce executable-line scope and run advisory mutation testing")) + assert.equal(workflow.match(/continue-on-error: true/g)?.length, 1) + assert.equal(workflow.match(/Could not write the job summary/g)?.length, 2) + const script = fs.readFileSync(path.join(repositoryRoot, "scripts/stryker-diff.mjs"), "utf8") + assert.ok(script.includes("appendSummary([], manifest.advisories, manifest)")) }) }) @@ -154,7 +160,7 @@ describe("buildManifest", () => { () => "@@ -1 +1 @@\n", ) - assert.deepEqual(manifest, { packages: [] }) + assert.deepEqual(manifest, { packages: [], advisories: [] }) }) it("fails rather than skipping a package over the changed-line cap", () => { @@ -169,6 +175,17 @@ describe("buildManifest", () => { /split the PR or obtain a maintainer-reviewed narrow exclusion/i, ) }) + + it("reports invalid mutation exclusions without bypassing executable-line accounting", () => { + const manifest = buildManifest( + [{ status: "A", path: "packages/core/src/value.ts" }], + () => "// Stryker disable next-line all: noisy\nexport const value = true\n", + () => "", + ) + + assert.equal(manifest.packages[0].changedExecutableLines, 1) + assert.match(manifest.advisories.join("\n"), /broad or unreasoned exclusions are not allowed/) + }) }) describe("packageForPath", () => { @@ -249,7 +266,6 @@ describe("shouldUseVitestRelated", () => { }) }) - describe("related-test discovery", () => { it("keeps Stryker's temp directory relative to each run root", () => { assert.equal(resolveStrykerTempDir("/repo", "/repo"), ".stryker-tmp") @@ -457,7 +473,7 @@ describe("failure output", () => { }, ] - it("lists every blocking mutant with tests, reproduction, exclusion, and report guidance", () => { + it("lists every advisory mutant with tests, reproduction, exclusion, and report guidance", () => { const baseSha = "a".repeat(40) const headSha = "b".repeat(40) const summary = formatSummary( @@ -475,10 +491,10 @@ describe("failure output", () => { survived: 2, noCoverage: 1, blocking, - result: "Failed", + result: "Advisory findings", }, ], - ["extension has blocking mutants"], + ["extension has advisory mutants"], { baseSha, headSha }, ) @@ -491,6 +507,7 @@ describe("failure output", () => { assert.ok(summary.includes("Stryker disable next-line ConditionalExpression:")) assert.ok(summary.includes("`reports/mutation/extension/mutation.html`")) assert.ok(summary.includes("`changed-code-mutation-report` artifact")) + assert.ok(summary.includes("### Advisory findings")) }) it("caps annotations without truncating the grouped summary", () => { @@ -548,7 +565,7 @@ describe("failure output", () => { assert.equal( command, - "::error file=src/value%3Aone%2Ctwo.ts,line=4,title=Mutation test gap::Survived mutant (replacement: left, right). 100%25 reproducible.", + "::warning file=src/value%3Aone%2Ctwo.ts,line=4,title=Mutation test advisory::Survived mutant (replacement: left, right). 100%25 reproducible.", ) }) @@ -558,30 +575,42 @@ describe("failure output", () => { try { fs.mkdirSync(path.join(repo, "packages/core"), { recursive: true }) - assert.throws( - () => - runManifest( - repo, - { - packages: [ - { - id: "core", - root: "packages/core", - vitestConfig: "vitest.unit.config.ts", - selectors: ["src/value.ts:1-1"], - changedExecutableLines: 1, - }, - ], - }, - reportRoot, - ), - /core Stryker preflight could not start:.*ENOENT/, + assert.doesNotThrow(() => + runManifest( + repo, + { + packages: [ + { + id: "core", + root: "packages/core", + vitestConfig: "vitest.unit.config.ts", + selectors: ["src/value.ts:1-1"], + changedExecutableLines: 1, + }, + ], + }, + reportRoot, + ), ) } finally { fs.rmSync(repo, { recursive: true, force: true }) } }) + it("does not fail when the GitHub job summary cannot be written", () => { + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-")) + process.env.GITHUB_STEP_SUMMARY = summaryDirectory + + try { + assert.doesNotThrow(() => appendSummary([], ["advisory"], {})) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + fs.rmSync(summaryDirectory, { recursive: true, force: true }) + } + }) + it("uses the actual tests recorded by Stryker", () => { assert.deepEqual( testsFromMutationReport({ testFiles: { "src/value.test.ts": {}, "src/other.spec.ts": {} } }, [ @@ -612,7 +641,7 @@ describe("failure output", () => { replacement: "x".repeat(1_000), location: { start: { line: 1 } }, })), - result: "Failed", + result: "Advisory findings", })) const summary = formatSummary(rows, ["mutation failure"], { baseSha: "a".repeat(40), @@ -627,7 +656,7 @@ describe("failure output", () => { describe("report evaluation", () => { const packageEntry = { id: "core", root: "packages/core" } - it("fails on surviving and uncovered changed-code mutants", () => { + it("reports surviving and uncovered changed-code mutants as advisory", () => { const report = { files: { "src/value.ts": { @@ -649,37 +678,39 @@ describe("report evaluation", () => { }, } - assert.throws(() => evaluateReport(report, packageEntry), /1 surviving and 1 uncovered/) + const result = evaluateReport(report, packageEntry) + assert.match(result.advisories.join("\n"), /1 surviving and 1 uncovered/) assert.equal(formatAnnotations(mutantCounts(report).blocking, packageEntry.root).length, 2) }) - it("passes only killed or timed-out mutants within the cap", () => { + it("has no advisories for killed or limited timed-out mutants within the cap", () => { const mutants = Array.from({ length: MAX_MUTANTS }, (_, index) => ({ status: index === 0 ? "Timeout" : "Killed", location: { start: { line: index + 1 } }, })) const counts = evaluateReport({ files: { "src/value.ts": { mutants } } }, packageEntry) assert.equal(counts.valid, MAX_MUTANTS) + assert.deepEqual(counts.advisories, []) }) - it("fails when valid mutants exceed the cap", () => { + it("reports valid mutants over the cap as advisory", () => { const mutants = Array.from({ length: MAX_MUTANTS + 1 }, (_, index) => ({ status: "Killed", location: { start: { line: index + 1 } }, })) - assert.throws( - () => evaluateReport({ files: { "src/value.ts": { mutants } } }, packageEntry), + assert.match( + evaluateReport({ files: { "src/value.ts": { mutants } } }, packageEntry).advisories.join("\n"), /split the PR or obtain a maintainer-reviewed narrow exclusion/i, ) }) - it("fails when timeouts could create false confidence", () => { + it("reports excessive timeouts as advisory", () => { const mutants = Array.from({ length: 10 }, (_, index) => ({ status: index < 2 ? "Timeout" : "Killed", location: { start: { line: index + 1 } }, })) - assert.throws( - () => evaluateReport({ files: { "src/value.ts": { mutants } } }, packageEntry), + assert.match( + evaluateReport({ files: { "src/value.ts": { mutants } } }, packageEntry).advisories.join("\n"), /result is inconclusive/, ) }) From f04b0c2313d7f2c49352a6c3d9c6a586fb98e6a5 Mon Sep 17 00:00:00 2001 From: Roomote Date: Sat, 12 Sep 2026 02:15:49 +0000 Subject: [PATCH 2/2] fix(ci): surface mutation advisories --- scripts/stryker-diff.mjs | 7 ++++++- scripts/stryker-diff.test.mjs | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/stryker-diff.mjs b/scripts/stryker-diff.mjs index 3fb7a34093..c3bf60e9db 100644 --- a/scripts/stryker-diff.mjs +++ b/scripts/stryker-diff.mjs @@ -454,6 +454,10 @@ export function formatAnnotationCommand(annotation) { return `::warning file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test advisory::${escapeWorkflowData(annotation.message)}` } +export function formatAdvisoryCommand(advisory) { + return `::warning title=Mutation test advisory::${escapeWorkflowData(advisory)}` +} + export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) { const annotations = [] @@ -578,7 +582,7 @@ export function formatSummary(rows, advisories, manifest = {}) { "const result = condition ? value : fallback", "```", "", - "Broad `all` exclusions and exclusions without a concrete reason are rejected by the gate.", + "Broad `all` exclusions and exclusions without a concrete reason are reported as advisories.", ) } @@ -622,6 +626,7 @@ export function formatSummary(rows, advisories, manifest = {}) { } export function appendSummary(rows, advisories, manifest) { + for (const advisory of advisories) console.warn(formatAdvisoryCommand(advisory)) if (!process.env.GITHUB_STEP_SUMMARY) return try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, advisories, manifest)) diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index 0e8d280667..5a1221471c 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -16,6 +16,7 @@ import { evaluateReport, executableChangedLines, formatAnnotations, + formatAdvisoryCommand, formatAnnotationCommand, formatBlockingMutants, formatSummary, @@ -569,6 +570,13 @@ describe("failure output", () => { ) }) + it("escapes aggregated advisory warnings", () => { + assert.equal( + formatAdvisoryCommand("preflight failed: 100%\nretry"), + "::warning title=Mutation test advisory::preflight failed: 100%25%0Aretry", + ) + }) + it("reports a Stryker preflight launch error when the binary is missing", () => { const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-launch-")) const reportRoot = path.join(repo, "reports") @@ -611,6 +619,23 @@ describe("failure output", () => { } }) + it("emits aggregated warnings when the GitHub job summary is unavailable", () => { + const previousSummary = process.env.GITHUB_STEP_SUMMARY + const previousWarn = console.warn + const warnings = [] + delete process.env.GITHUB_STEP_SUMMARY + console.warn = (warning) => warnings.push(warning) + + try { + appendSummary([], ["manifest invalid\nreview it"], {}) + assert.deepEqual(warnings, ["::warning title=Mutation test advisory::manifest invalid%0Areview it"]) + } finally { + if (previousSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previousSummary + console.warn = previousWarn + } + }) + it("uses the actual tests recorded by Stryker", () => { assert.deepEqual( testsFromMutationReport({ testFiles: { "src/value.test.ts": {}, "src/other.spec.ts": {} } }, [