[Improve] Make mutation warnings easier to review

scripts/stryker-diff.mjs

@@ -460,26 +460,36 @@ export function formatAdvisoryCommand(advisory) {
 
 export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) {
 	const annotations = []
+	const mutantsByLocation = new Map()
 
-	for (const mutant of blockingMutants.sort((left, right) => {
+	for (const mutant of [...blockingMutants].sort((left, right) => {
 		const pathOrder = left.filePath.localeCompare(right.filePath)
 		return pathOrder || left.location.start.line - right.location.start.line
 	})) {
 		const repositoryPath = path.posix.join(packageRoot, mutant.filePath.replaceAll("\\", "/"))
 		const key = `${repositoryPath}:${mutant.location.start.line}`
+		const group = mutantsByLocation.get(key) ?? { repositoryPath, mutants: [] }
+		group.mutants.push(mutant)
+		mutantsByLocation.set(key, group)
+	}
+
+	for (const [key, { repositoryPath, mutants }] of mutantsByLocation) {
 		const fileCount = state.perFile.get(repositoryPath) ?? 0
-		if (annotations.some((annotation) => annotation.key === key) || fileCount >= 7 || state.total >= 20) continue
+		if (fileCount >= 7 || state.total >= 20) continue
 
+		const mutant = mutants[0]
 		const replacement = String(mutant.replacement ?? "")
 			.replace(/\s+/g, " ")
 			.trim()
 			.slice(0, 160)
+		const location = `${repositoryPath}:${mutant.location.start.line}`
+		const detail = `${mutant.status} ${mutant.mutatorName} mutant${replacement ? ` (replacement: ${replacement})` : ""}`
 		annotations.push({
 			key,
 			file: repositoryPath,
 			line: mutant.location.start.line,
 			message:
-				`${mutant.status} ${mutant.mutatorName} mutant${replacement ? ` (replacement: ${replacement})` : ""}. ` +
+				`${location}: ${mutants.length === 1 ? detail : `${mutants.length} mutation test gaps; example: ${detail}`}. ` +
 				"See the job summary for the complete list and resolution guidance.",
 		})
 		state.perFile.set(repositoryPath, fileCount + 1)
@@ -652,16 +662,18 @@ export function evaluateReport(report, packageEntry) {
 				"The result is inconclusive; consider fixing flaky or slow tests, or reducing the changed scope.",
 		)
 	}
-	if (counts.blocking.length > 0) {
-		advisories.push(
-			`${packageEntry.id} has ${counts.survived} surviving and ${counts.noCoverage} uncovered changed-code mutants. ` +
-				"Consider adding or strengthening focused tests.",
-		)
-	}
 	return { ...counts, advisories }
 }
 
-export function runManifest(repoRoot, manifest, reportRoot) {
+export function runManifest(
+	repoRoot,
+	manifest,
+	reportRoot,
+	{
+		runMutation = runStryker,
+		readMutationReport = (reportPath) => JSON.parse(fs.readFileSync(reportPath, "utf8")),
+	} = {},
+) {
 	const rows = []
 	const advisories = [...(manifest.advisories ?? [])]
 	const annotationState = { total: 0, perFile: new Map() }
@@ -677,7 +689,7 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 			if (packageEntry.discoverRelatedTests) {
 				packageEntry.testFiles = discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory)
 			}
-			const preflightOutput = stripAnsi(runStryker(repoRoot, packageEntry, reportRoot, true))
+			const preflightOutput = stripAnsi(runMutation(repoRoot, packageEntry, reportRoot, true))
 			const mutantMatch = /Instrumented \d+ source file\(s\) with (\d+) mutant\(s\)/.exec(preflightOutput)
 			if (!mutantMatch) throw new Error(`${packageEntry.id} preflight did not report a mutant count`)
 			const generatedMutants = Number(mutantMatch[1])
@@ -707,9 +719,9 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 				continue
 			}
 
-			runStryker(repoRoot, packageEntry, reportRoot, false)
+			runMutation(repoRoot, packageEntry, reportRoot, false)
 			const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json")
-			const report = JSON.parse(fs.readFileSync(jsonReportPath, "utf8"))
+			const report = readMutationReport(jsonReportPath)
 			packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles)
 			counts = evaluateReport(report, packageEntry)
 			for (const annotation of formatAnnotations(
@@ -729,7 +741,7 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 				reportPath,
 				changedLines: packageEntry.changedExecutableLines,
 				...counts,
-				result: counts.advisories.length > 0 ? "Advisory findings" : "Passed",
+				result: counts.blocking.length > 0 || counts.advisories.length > 0 ? "Advisory findings" : "Passed",
 			})
 		} catch (error) {
 			advisories.push(error.message)

scripts/stryker-diff.test.mjs

@@ -529,6 +529,43 @@ describe("failure output", () => {
 		for (const mutant of manyMutants) assert.ok(grouped.includes(mutant.mutatorName))
 	})
 
+	it("emits one distinguishable annotation per source location", () => {
+		const mutants = [
+			blocking[2],
+			blocking[1],
+			{
+				filePath: "utils/other.ts",
+				status: "NoCoverage",
+				mutatorName: "ConditionalExpression",
+				replacement: "true",
+				location: { start: { line: 9 } },
+			},
+			blocking[0],
+		]
+		const originalOrder = [...mutants]
+		const annotations = formatAnnotations(mutants, "src")
+
+		assert.equal(annotations.length, 2)
+		assert.deepEqual(mutants, originalOrder)
+		assert.match(
+			annotations[0].message,
+			/^src\/core\/value\.ts:4: 2 mutation test gaps; example: NoCoverage StringLiteral mutant \(replacement: "left \| right"\)/,
+		)
+		assert.match(
+			annotations[1].message,
+			/^src\/utils\/other\.ts:9: 2 mutation test gaps; example: Survived BooleanLiteral mutant \(replacement: false\)/,
+		)
+	})
+
+	it("prefixes singleton annotations with their source location", () => {
+		const [annotation] = formatAnnotations([blocking[2]], "src")
+
+		assert.equal(
+			annotation.message,
+			"src/utils/other.ts:9: Survived BooleanLiteral mutant (replacement: false). See the job summary for the complete list and resolution guidance.",
+		)
+	})
+
 	it("shares annotation limits across packages", () => {
 		const state = { total: 0, perFile: new Map() }
 		const first = formatAnnotations(
@@ -605,6 +642,50 @@ describe("failure output", () => {
 		}
 	})
 
+	it("classifies successful package rows from blocking mutants", () => {
+		const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-success-"))
+		const packageEntry = {
+			id: "core",
+			root: "packages/core",
+			vitestConfig: "vitest.unit.config.ts",
+			selectors: ["src/value.ts:1-1"],
+			changedExecutableLines: 1,
+		}
+		const execute = (report) =>
+			runManifest(repo, { packages: [{ ...packageEntry }] }, path.join(repo, "reports"), {
+				runMutation: (_repoRoot, _entry, _reportRoot, dryRunOnly) =>
+					dryRunOnly ? "Instrumented 1 source file(s) with 1 mutant(s)" : "",
+				readMutationReport: () => report,
+			})[0]
+
+		try {
+			const advisoryRow = execute({
+				files: {
+					"src/value.ts": {
+						mutants: [
+							{
+								status: "Survived",
+								mutatorName: "BooleanLiteral",
+								replacement: "false",
+								location: { start: { line: 1 } },
+							},
+						],
+					},
+				},
+			})
+			assert.equal(advisoryRow.result, "Advisory findings")
+			assert.deepEqual(advisoryRow.advisories, [])
+
+			const passedRow = execute({
+				files: { "src/value.ts": { mutants: [{ status: "Killed", location: { start: { line: 1 } } }] } },
+			})
+			assert.equal(passedRow.result, "Passed")
+			assert.deepEqual(passedRow.advisories, [])
+		} finally {
+			fs.rmSync(repo, { recursive: true, force: true })
+		}
+	})
+
 	it("does not fail when the GitHub job summary cannot be written", () => {
 		const previousSummary = process.env.GITHUB_STEP_SUMMARY
 		const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-"))
@@ -681,7 +762,7 @@ describe("failure output", () => {
 describe("report evaluation", () => {
 	const packageEntry = { id: "core", root: "packages/core" }
 
-	it("reports surviving and uncovered changed-code mutants as advisory", () => {
+	it("reports surviving and uncovered mutants through detailed annotations without a redundant aggregate", () => {
 		const report = {
 			files: {
 				"src/value.ts": {
@@ -704,7 +785,7 @@ describe("report evaluation", () => {
 		}
 
 		const result = evaluateReport(report, packageEntry)
-		assert.match(result.advisories.join("\n"), /1 surviving and 1 uncovered/)
+		assert.deepEqual(result.advisories, [])
 		assert.equal(formatAnnotations(mutantCounts(report).blocking, packageEntry.root).length, 2)
 	})