diff --git a/.github/workflows/mutation-report.yml b/.github/workflows/mutation-report.yml index 5707c35..c920b9d 100644 --- a/.github/workflows/mutation-report.yml +++ b/.github/workflows/mutation-report.yml @@ -51,6 +51,7 @@ jobs: - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0 - name: Mutate the changed Go + id: go if: ${{ inputs.go-directory != '' }} continue-on-error: true working-directory: ${{ inputs.go-directory }} @@ -59,9 +60,14 @@ jobs: GREMLINS: ${{ inputs.gremlins-version }} run: | go install "github.com/go-gremlins/gremlins/cmd/gremlins@$GREMLINS" - "$(go env GOPATH)/bin/gremlins" unleash --diff "$BASE" --output "$RUNNER_TEMP/gremlins.json" ./... + bin=$(go env GOBIN) + # gremlins reads the module root as `.`; given `./...` it finds no mutants at all. Each + # mutant reruns the tests, and the default timeout times out a race-enabled suite. + "${bin:-$(go env GOPATH)/bin}/gremlins" unleash --diff "$BASE" --timeout-coefficient 40 \ + --output "$RUNNER_TEMP/gremlins.json" . - name: Mutate the changed TypeScript + id: web if: ${{ inputs.web-directory != '' }} continue-on-error: true working-directory: ${{ inputs.web-directory }} @@ -75,14 +81,22 @@ jobs: exit 0 fi pnpm install --frozen-lockfile - pnpm exec stryker run --testRunner vitest --reporters json,clear-text \ - --jsonReporter.fileName "$RUNNER_TEMP/mutation.json" --mutate "$changed" + # pnpm keeps each package's dependencies apart, so Stryker is told where its runner is. + pnpm exec stryker run --plugins @stryker-mutator/vitest-runner --testRunner vitest \ + --reporters json,clear-text --mutate "$changed" + cp reports/mutation/mutation.json "$RUNNER_TEMP/mutation.json" - name: Write the report + env: + GO_OUTCOME: ${{ steps.go.outcome }} + WEB_OUTCOME: ${{ steps.web.outcome }} run: | + failed=() + [ "$GO_OUTCOME" = failure ] && failed+=(--failed gremlins) + [ "$WEB_OUTCOME" = failure ] && failed+=(--failed Stryker) python3 .github-workflows/scripts/mutation_report.py \ --gremlins "$RUNNER_TEMP/gremlins.json" --stryker "$RUNNER_TEMP/mutation.json" \ - --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" + "${failed[@]}" --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" - name: Comment on the pull request env: diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py index 431e3ee..c428002 100644 --- a/scripts/mutation_report.py +++ b/scripts/mutation_report.py @@ -8,7 +8,9 @@ mutation_report.py --gremlins gremlins.json --stryker mutation.json --out comment.md A report that is absent counts as no mutants, so a repository with only Go or -only TypeScript passes one flag. +only TypeScript passes one flag. A tool that crashed also leaves no report, so the +workflow names it with --failed, and the comment says so instead of claiming every +mutant was caught. """ from __future__ import annotations @@ -63,7 +65,9 @@ def _table(mutants: list[Mutant]) -> list[str]: return rows -def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None) -> str: +def render( + survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None, failed: list[str] | None = None +) -> str: lines = [MARKER, "### Mutation report", ""] if survived: noun = "mutant" if len(survived) == 1 else "mutants" @@ -73,8 +77,10 @@ def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str] "", *_table(survived), ] - else: + elif not failed: lines.append("Every mutant on the changed lines was caught by a test.") + for tool in failed or []: + lines += ["", f"{tool} did not finish, so its mutants are not listed: see the job log."] if unreached: lines += ["", "#### No test reaches these lines", "", *_table(unreached)] for problem in problems or []: @@ -98,12 +104,13 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--gremlins", help="gremlins' JSON report (--output)") parser.add_argument("--stryker", help="Stryker's JSON report (json reporter)") parser.add_argument("--out", required=True, help="where to write the comment") + parser.add_argument("--failed", action="append", default=[], help="a tool whose run did not finish") args = parser.parse_args(argv) problems: list[str] = [] g_survived, g_unreached = from_gremlins(_read(args.gremlins, "gremlins", problems)) s_survived, s_unreached = from_stryker(_read(args.stryker, "Stryker", problems)) - comment = render(g_survived + s_survived, g_unreached + s_unreached, problems) + comment = render(g_survived + s_survived, g_unreached + s_unreached, problems, args.failed) Path(args.out).write_text(comment) print(comment) return 0 diff --git a/tests/test_mutation_report.py b/tests/test_mutation_report.py index 4d258c4..5f4a6e0 100644 --- a/tests/test_mutation_report.py +++ b/tests/test_mutation_report.py @@ -108,6 +108,16 @@ def test_a_missing_report_counts_as_no_mutants(self) -> None: self.assertEqual(code, 0) self.assertIn("Every mutant on the changed lines was caught by a test.", out.read_text()) + def test_a_tool_that_did_not_finish_is_named_instead_of_reading_as_all_caught(self) -> None: + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) / "comment.md" + with redirect_stdout(io.StringIO()): + code = main(["--failed", "Stryker", "--out", str(out)]) + text = out.read_text() + self.assertEqual(code, 0) + self.assertIn("Stryker did not finish, so its mutants are not listed: see the job log.", text) + self.assertNotIn("Every mutant on the changed lines was caught", text) + def test_a_report_that_is_not_json_is_named_but_does_not_fail(self) -> None: with tempfile.TemporaryDirectory() as directory: broken, out = Path(directory) / "g.json", Path(directory) / "comment.md"