feat: a mutation report on the lines a pull request changes

.github/workflows/ci.yml

@@ -106,6 +106,14 @@ jobs:
           .venv-tests/bin/python -m coverage run --branch --source=scripts.path_buckets -m unittest tests.test_path_buckets
           .venv-tests/bin/python -m coverage report --show-missing --fail-under=100
 
+      # The mutation report is what reviewers read to judge a pull request's tests, so every line and
+      # branch of it is held too.
+      - name: Run mutation report tests with branch coverage
+        if: ${{ !cancelled() }}
+        run: |
+          .venv-tests/bin/python -m coverage run --branch --source=scripts.mutation_report -m unittest tests.test_mutation_report
+          .venv-tests/bin/python -m coverage report --show-missing --fail-under=100
+
   # ---------------------------------------------------------------------------
   # Terminal aggregator. This is the single required check in the ruleset.
   # It runs even when an upstream job fails/cancels (if: always()) and then

.github/workflows/mutation-report.yml

@@ -0,0 +1,99 @@
+name: Mutation Report
+
+# Reports, on a pull request, the mutants that survive on the lines it changes: a test ran the
+# line, yet no test failed when it changed. gremlins mutates Go, Stryker mutates TypeScript and
+# Vue. One comment is kept up to date per pull request. It is a report and never fails the build
+# (workspace docs/testing-standards.md, section 6).
+#
+# The caller's toolchain comes from its mise.toml. A web directory needs @stryker-mutator/core and
+# @stryker-mutator/vitest-runner as devDependencies, so Stryker runs with the project's own vitest.
+
+on:
+  workflow_call:
+    inputs:
+      go-directory:
+        description: Directory of the Go module to mutate; empty skips Go.
+        type: string
+        default: ''
+      web-directory:
+        description: Directory of the web project to mutate; empty skips TypeScript.
+        type: string
+        default: ''
+      gremlins-version:
+        description: gremlins release to install.
+        type: string
+        default: v0.6.0
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  report:
+    name: Mutation report
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      # Checkout github-workflows at this workflow file's own released version; release-please
+      # keeps the literal in step with the tag this file ships in.
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: JorisJonkers-dev/github-workflows
+          ref: v0.24.0 # x-release-please-version
+          path: .github-workflows
+          persist-credentials: false
+
+      - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0
+
+      - name: Mutate the changed Go
+        if: ${{ inputs.go-directory != '' }}
+        continue-on-error: true
+        working-directory: ${{ inputs.go-directory }}
+        env:
+          BASE: origin/${{ github.base_ref }}
+          GREMLINS: ${{ inputs.gremlins-version }}
+        run: |
+          go install "github.com/go-gremlins/gremlins/cmd/gremlins@$GREMLINS"
+          "$(go env GOPATH)/bin/gremlins" unleash --diff "$BASE" --output "$RUNNER_TEMP/gremlins.json" ./...
+
+      - name: Mutate the changed TypeScript
+        if: ${{ inputs.web-directory != '' }}
+        continue-on-error: true
+        working-directory: ${{ inputs.web-directory }}
+        env:
+          BASE: origin/${{ github.base_ref }}
+        run: |
+          changed=$(git diff --name-only --diff-filter=AM --relative "$BASE"...HEAD -- src \
+            | grep -E '\.(ts|vue)$' | grep -Ev '\.(spec|test)\.ts$|\.gen\.ts$|/infrastructure/api/' | paste -sd, - || true)
+          if [ -z "$changed" ]; then
+            echo "No changed TypeScript or Vue source to mutate."
+            exit 0
+          fi
+          pnpm install --frozen-lockfile
+          pnpm exec stryker run --testRunner vitest --reporters json,clear-text \
+            --jsonReporter.fileName "$RUNNER_TEMP/mutation.json" --mutate "$changed"
+
+      - name: Write the report
+        run: |
+          python3 .github-workflows/scripts/mutation_report.py \
+            --gremlins "$RUNNER_TEMP/gremlins.json" --stryker "$RUNNER_TEMP/mutation.json" \
+            --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment on the pull request
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+        run: |
+          existing=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
+            --jq '[.[] | select(.body | startswith("<!-- mutation-report -->"))][0].id // empty')
+          if [ -n "$existing" ]; then
+            gh api --method PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$RUNNER_TEMP/comment.md" >/dev/null
+          else
+            gh api --method POST "repos/$REPO/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" >/dev/null
+          fi

README.md

@@ -34,6 +34,7 @@ should call released tags instead of branches.
 | `python-ci.yml` | Python Ruff, typecheck, and test workflow with overrideable commands. |
 | `nix-ci.yml` | Nix formatter, Statix, Deadnix, flake check, and optional validation. |
 | `jvm-ci.yml` | Gradle static lint/test workflow that fails Gradle warnings and deprecations. |
+| `mutation-report.yml` | Mutates the lines a pull request changes (gremlins for Go, Stryker for TypeScript and Vue) and keeps one comment listing the mutants that survived. Never fails the build. |
 | `docker-image-ci.yml` | Build a Docker image without publishing. |
 | `container-publish.yml` | Publish a GHCR image with release and sha tags. |
 | `publish-api-clients.yml` | Generate and publish API clients from an OpenAPI spec. |

release-please-config.json

@@ -54,6 +54,10 @@
           "type": "generic",
           "path": ".github/workflows/migration-guard.yml"
         },
+        {
+          "type": "generic",
+          "path": ".github/workflows/mutation-report.yml"
+        },
         {
           "type": "generic",
           "path": ".github/workflows/node-ci.yml"

scripts/mutation_report.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Turn gremlins' and Stryker's JSON reports into one pull request comment.
+
+Both tools run only on the lines a pull request changes. The comment lists every
+mutant that survived (a test ran the line but no test failed when it changed) and,
+apart, every mutant no test reaches. It is a report: it never fails the build.
+
+    mutation_report.py --gremlins gremlins.json --stryker mutation.json --out comment.md
+
+A report that is absent counts as no mutants, so a repository with only Go or
+only TypeScript passes one flag.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+# The comment is found again by this line, so a later run edits it instead of adding another.
+MARKER = "<!-- mutation-report -->"
+
+
+@dataclass(frozen=True, order=True)
+class Mutant:
+    file: str
+    line: int
+    mutator: str
+
+
+def from_gremlins(report: dict) -> tuple[list[Mutant], list[Mutant]]:
+    """The mutants gremlins found LIVED, and those NOT COVERED."""
+    survived: list[Mutant] = []
+    unreached: list[Mutant] = []
+    for f in report.get("files") or []:
+        for m in f.get("mutations") or []:
+            mutant = Mutant(f["file_name"], m["line"], m["type"])
+            if m["status"] == "LIVED":
+                survived.append(mutant)
+            elif m["status"] == "NOT COVERED":
+                unreached.append(mutant)
+    return survived, unreached
+
+
+def from_stryker(report: dict) -> tuple[list[Mutant], list[Mutant]]:
+    """The mutants Stryker marked Survived, and those with NoCoverage."""
+    survived: list[Mutant] = []
+    unreached: list[Mutant] = []
+    for path, f in (report.get("files") or {}).items():
+        for m in f.get("mutants") or []:
+            mutant = Mutant(path, m["location"]["start"]["line"], m["mutatorName"])
+            if m["status"] == "Survived":
+                survived.append(mutant)
+            elif m["status"] == "NoCoverage":
+                unreached.append(mutant)
+    return survived, unreached
+
+
+def _table(mutants: list[Mutant]) -> list[str]:
+    rows = ["| where | mutation |", "|---|---|"]
+    rows += [f"| `{m.file}:{m.line}` | {m.mutator} |" for m in sorted(mutants)]
+    return rows
+
+
+def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None) -> str:
+    lines = [MARKER, "### Mutation report", ""]
+    if survived:
+        noun = "mutant" if len(survived) == 1 else "mutants"
+        lines += [
+            f"{len(survived)} {noun} survived on the changed lines: a test ran the line, and no test failed "
+            "when it changed. Each is a check a test is missing.",
+            "",
+            *_table(survived),
+        ]
+    else:
+        lines.append("Every mutant on the changed lines was caught by a test.")
+    if unreached:
+        lines += ["", "#### No test reaches these lines", "", *_table(unreached)]
+    for problem in problems or []:
+        lines += ["", f"- {problem}"]
+    lines += ["", "This report never fails the build."]
+    return "\n".join(lines) + "\n"
+
+
+def _read(path: str | None, name: str, problems: list[str]) -> dict:
+    if not path or not Path(path).is_file():
+        return {}
+    try:
+        return json.loads(Path(path).read_text())
+    except ValueError:
+        problems.append(f"The {name} report could not be read, so its mutants are not listed.")
+        return {}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Turn mutation reports into one pull request comment.")
+    parser.add_argument("--gremlins", help="gremlins' JSON report (--output)")
+    parser.add_argument("--stryker", help="Stryker's JSON report (json reporter)")
+    parser.add_argument("--out", required=True, help="where to write the comment")
+    args = parser.parse_args(argv)
+
+    problems: list[str] = []
+    g_survived, g_unreached = from_gremlins(_read(args.gremlins, "gremlins", problems))
+    s_survived, s_unreached = from_stryker(_read(args.stryker, "Stryker", problems))
+    comment = render(g_survived + s_survived, g_unreached + s_unreached, problems)
+    Path(args.out).write_text(comment)
+    print(comment)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

tests/test_mutation_report.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+
+from scripts.mutation_report import MARKER, Mutant, from_gremlins, from_stryker, main, render
+
+GREMLINS = {
+    "go_module": "github.com/example/app",
+    "files": [
+        {
+            "file_name": "internal/access/app/service.go",
+            "mutations": [
+                {"type": "CONDITIONALS_BOUNDARY", "status": "LIVED", "line": 42, "column": 7},
+                {"type": "INVERT_NEGATIVES", "status": "KILLED", "line": 50, "column": 3},
+                {"type": "ARITHMETIC_BASE", "status": "NOT COVERED", "line": 61, "column": 9},
+            ],
+        }
+    ],
+}
+
+STRYKER = {
+    "schemaVersion": "1",
+    "files": {
+        "src/features/home/useLauncher.ts": {
+            "language": "typescript",
+            "mutants": [
+                {"id": "1", "mutatorName": "ConditionalExpression", "status": "Survived", "location": {"start": {"line": 14, "column": 5}}},
+                {"id": "2", "mutatorName": "StringLiteral", "status": "Killed", "location": {"start": {"line": 15, "column": 1}}},
+                {"id": "3", "mutatorName": "BooleanLiteral", "status": "NoCoverage", "location": {"start": {"line": 20, "column": 2}}},
+            ],
+        }
+    },
+}
+
+
+class ReadingGremlins(unittest.TestCase):
+    def test_a_mutant_that_lived_is_a_survivor(self) -> None:
+        survived, _ = from_gremlins(GREMLINS)
+        self.assertEqual(survived, [Mutant("internal/access/app/service.go", 42, "CONDITIONALS_BOUNDARY")])
+
+    def test_a_mutant_no_test_reaches_is_reported_apart(self) -> None:
+        _, unreached = from_gremlins(GREMLINS)
+        self.assertEqual(unreached, [Mutant("internal/access/app/service.go", 61, "ARITHMETIC_BASE")])
+
+    def test_a_run_with_no_changed_go_reports_nothing(self) -> None:
+        self.assertEqual(from_gremlins({"files": None}), ([], []))
+
+
+class ReadingStryker(unittest.TestCase):
+    def test_a_surviving_mutant_is_a_survivor(self) -> None:
+        survived, _ = from_stryker(STRYKER)
+        self.assertEqual(survived, [Mutant("src/features/home/useLauncher.ts", 14, "ConditionalExpression")])
+
+    def test_a_mutant_without_coverage_is_reported_apart(self) -> None:
+        _, unreached = from_stryker(STRYKER)
+        self.assertEqual(unreached, [Mutant("src/features/home/useLauncher.ts", 20, "BooleanLiteral")])
+
+
+class Rendering(unittest.TestCase):
+    def test_the_comment_carries_the_marker_it_is_found_by(self) -> None:
+        self.assertTrue(render([], []).startswith(MARKER))
+
+    def test_no_survivors_says_every_mutant_was_caught(self) -> None:
+        self.assertIn("Every mutant on the changed lines was caught by a test.", render([], []))
+
+    def test_each_survivor_is_listed_with_where_it_lives(self) -> None:
+        text = render([Mutant("a.go", 3, "INVERT_NEGATIVES")], [])
+        self.assertIn("| `a.go:3` | INVERT_NEGATIVES |", text)
+        self.assertIn("1 mutant survived", text)
+
+    def test_unreached_mutants_get_their_own_section(self) -> None:
+        text = render([], [Mutant("b.ts", 9, "StringLiteral")])
+        self.assertIn("No test reaches these lines", text)
+        self.assertIn("| `b.ts:9` | StringLiteral |", text)
+
+    def test_the_comment_says_it_never_fails_the_build(self) -> None:
+        self.assertIn("This report never fails the build.", render([], []))
+
+    def test_survivors_are_sorted_by_file_and_line(self) -> None:
+        text = render([Mutant("b.go", 1, "X"), Mutant("a.go", 9, "Y"), Mutant("a.go", 2, "Z")], [])
+        self.assertLess(text.index("`a.go:2`"), text.index("`a.go:9`"))
+        self.assertLess(text.index("`a.go:9`"), text.index("`b.go:1`"))
+
+
+class CommandLine(unittest.TestCase):
+    def test_it_writes_one_comment_from_both_reports(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            gremlins, stryker, out = (Path(directory) / name for name in ("g.json", "s.json", "comment.md"))
+            gremlins.write_text(json.dumps(GREMLINS))
+            stryker.write_text(json.dumps(STRYKER))
+            with redirect_stdout(io.StringIO()):
+                code = main(["--gremlins", str(gremlins), "--stryker", str(stryker), "--out", str(out)])
+            text = out.read_text()
+        self.assertEqual(code, 0)
+        self.assertIn("`internal/access/app/service.go:42`", text)
+        self.assertIn("`src/features/home/useLauncher.ts:14`", text)
+
+    def test_a_missing_report_counts_as_no_mutants(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            out = Path(directory) / "comment.md"
+            with redirect_stdout(io.StringIO()):
+                code = main(["--gremlins", str(Path(directory) / "absent.json"), "--out", str(out)])
+            self.assertEqual(code, 0)
+            self.assertIn("Every mutant on the changed lines was caught by a test.", out.read_text())
+
+    def test_a_report_that_is_not_json_is_named_but_does_not_fail(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            broken, out = Path(directory) / "g.json", Path(directory) / "comment.md"
+            broken.write_text("not json")
+            with redirect_stdout(io.StringIO()):
+                code = main(["--gremlins", str(broken), "--out", str(out)])
+            self.assertEqual(code, 0)
+            self.assertIn("could not be read", out.read_text())
+
+
+if __name__ == "__main__":
+    unittest.main()