fix: the mutation report runs gremlins and Stryker the way each actually works

.github/workflows/mutation-report.yml

@@ -51,6 +51,7 @@ jobs:
       - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0
 
       - name: Mutate the changed Go
+        id: go
         if: ${{ inputs.go-directory != '' }}
         continue-on-error: true
         working-directory: ${{ inputs.go-directory }}
@@ -59,9 +60,14 @@ jobs:
           GREMLINS: ${{ inputs.gremlins-version }}
         run: |
           go install "github.com/go-gremlins/gremlins/cmd/gremlins@$GREMLINS"
-          "$(go env GOPATH)/bin/gremlins" unleash --diff "$BASE" --output "$RUNNER_TEMP/gremlins.json" ./...
+          bin=$(go env GOBIN)
+          # gremlins reads the module root as `.`; given `./...` it finds no mutants at all. Each
+          # mutant reruns the tests, and the default timeout times out a race-enabled suite.
+          "${bin:-$(go env GOPATH)/bin}/gremlins" unleash --diff "$BASE" --timeout-coefficient 40 \
+            --output "$RUNNER_TEMP/gremlins.json" .
 
       - name: Mutate the changed TypeScript
+        id: web
         if: ${{ inputs.web-directory != '' }}
         continue-on-error: true
         working-directory: ${{ inputs.web-directory }}
@@ -75,14 +81,22 @@ jobs:
             exit 0
           fi
           pnpm install --frozen-lockfile
-          pnpm exec stryker run --testRunner vitest --reporters json,clear-text \
-            --jsonReporter.fileName "$RUNNER_TEMP/mutation.json" --mutate "$changed"
+          # pnpm keeps each package's dependencies apart, so Stryker is told where its runner is.
+          pnpm exec stryker run --plugins @stryker-mutator/vitest-runner --testRunner vitest \
+            --reporters json,clear-text --mutate "$changed"
+          cp reports/mutation/mutation.json "$RUNNER_TEMP/mutation.json"
 
       - name: Write the report
+        env:
+          GO_OUTCOME: ${{ steps.go.outcome }}
+          WEB_OUTCOME: ${{ steps.web.outcome }}
         run: |
+          failed=()
+          [ "$GO_OUTCOME" = failure ] && failed+=(--failed gremlins)
+          [ "$WEB_OUTCOME" = failure ] && failed+=(--failed Stryker)
           python3 .github-workflows/scripts/mutation_report.py \
             --gremlins "$RUNNER_TEMP/gremlins.json" --stryker "$RUNNER_TEMP/mutation.json" \
-            --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY"
+            "${failed[@]}" --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment on the pull request
         env:

scripts/mutation_report.py

@@ -8,7 +8,9 @@
     mutation_report.py --gremlins gremlins.json --stryker mutation.json --out comment.md
 
 A report that is absent counts as no mutants, so a repository with only Go or
-only TypeScript passes one flag.
+only TypeScript passes one flag. A tool that crashed also leaves no report, so the
+workflow names it with --failed, and the comment says so instead of claiming every
+mutant was caught.
 """
 
 from __future__ import annotations
@@ -63,7 +65,9 @@ def _table(mutants: list[Mutant]) -> list[str]:
     return rows
 
 
-def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None) -> str:
+def render(
+    survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None, failed: list[str] | None = None
+) -> str:
     lines = [MARKER, "### Mutation report", ""]
     if survived:
         noun = "mutant" if len(survived) == 1 else "mutants"
@@ -73,8 +77,10 @@ def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str]
             "",
             *_table(survived),
         ]
-    else:
+    elif not failed:
         lines.append("Every mutant on the changed lines was caught by a test.")
+    for tool in failed or []:
+        lines += ["", f"{tool} did not finish, so its mutants are not listed: see the job log."]
     if unreached:
         lines += ["", "#### No test reaches these lines", "", *_table(unreached)]
     for problem in problems or []:
@@ -98,12 +104,13 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--gremlins", help="gremlins' JSON report (--output)")
     parser.add_argument("--stryker", help="Stryker's JSON report (json reporter)")
     parser.add_argument("--out", required=True, help="where to write the comment")
+    parser.add_argument("--failed", action="append", default=[], help="a tool whose run did not finish")
     args = parser.parse_args(argv)
 
     problems: list[str] = []
     g_survived, g_unreached = from_gremlins(_read(args.gremlins, "gremlins", problems))
     s_survived, s_unreached = from_stryker(_read(args.stryker, "Stryker", problems))
-    comment = render(g_survived + s_survived, g_unreached + s_unreached, problems)
+    comment = render(g_survived + s_survived, g_unreached + s_unreached, problems, args.failed)
     Path(args.out).write_text(comment)
     print(comment)
     return 0

tests/test_mutation_report.py

@@ -108,6 +108,16 @@ def test_a_missing_report_counts_as_no_mutants(self) -> None:
             self.assertEqual(code, 0)
             self.assertIn("Every mutant on the changed lines was caught by a test.", out.read_text())
 
+    def test_a_tool_that_did_not_finish_is_named_instead_of_reading_as_all_caught(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            out = Path(directory) / "comment.md"
+            with redirect_stdout(io.StringIO()):
+                code = main(["--failed", "Stryker", "--out", str(out)])
+            text = out.read_text()
+        self.assertEqual(code, 0)
+        self.assertIn("Stryker did not finish, so its mutants are not listed: see the job log.", text)
+        self.assertNotIn("Every mutant on the changed lines was caught", text)
+
     def test_a_report_that_is_not_json_is_named_but_does_not_fail(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
             broken, out = Path(directory) / "g.json", Path(directory) / "comment.md"