diff --git a/.github/workflows/quality-advisory.yml b/.github/workflows/quality-advisory.yml index c982483b..9c910783 100644 --- a/.github/workflows/quality-advisory.yml +++ b/.github/workflows/quality-advisory.yml @@ -321,10 +321,23 @@ jobs: tail -c 20000 mutmut-run.txt if [ "$MUTMUT_OK" = "1" ]; then mutmut results > mutmut-results.txt 2>&1 || true - KILLED="$(grep -c ': killed' mutmut-results.txt || true)" + # `mutmut results` lists ONLY the mutants worth looking at -- survived / no tests / + # timeout / suspicious. Killed mutants are NEVER listed, so `grep -c ': killed'` is + # always 0 and a naive table reports "Killed 0" on a perfectly healthy run. Derive it: + # the run's final progress line carries the total (`461/461`), and every non-killed + # mutant is exactly one line in the results file. Verified against run 30308667584: + # 461 total - 374 listed = 87 killed, matching mutmut's own counter. + TOTAL="$(tr '\r' '\n' < mutmut-run.txt | grep -oE '[0-9]+/[0-9]+' | tail -1 | cut -d/ -f2)" + LISTED="$(grep -cE '^[[:space:]]+\S+: ' mutmut-results.txt || true)" SURVIVED="$(grep -c ': survived' mutmut-results.txt || true)" NOTESTS="$(grep -c ': no tests' mutmut-results.txt || true)" - echo "== killed=$KILLED survived=$SURVIVED no-tests=$NOTESTS ==" + if [ -n "$TOTAL" ] && [ "$TOTAL" -ge "$LISTED" ] 2>/dev/null; then + KILLED=$((TOTAL - LISTED)) + else + # Never invent a number: if the total could not be read, say so rather than print 0. + KILLED="?" + fi + echo "== total=$TOTAL killed=$KILLED survived=$SURVIVED no-tests=$NOTESTS ==" grep ': survived' mutmut-results.txt || true { echo "## Mutation testing — \`messagefoundry/parsing/binary.py\`" diff --git a/docs/quality-gates/HANDOFF-mutation-coverage.md b/docs/quality-gates/HANDOFF-mutation-coverage.md index 990d9850..84415e77 100644 --- a/docs/quality-gates/HANDOFF-mutation-coverage.md +++ b/docs/quality-gates/HANDOFF-mutation-coverage.md @@ -267,6 +267,12 @@ behaviour on fork PRs. The workflow still holds **no write scope on any job**. by the scoped test — in 3 seconds.** The step summary now carries that table plus the survivor list, and a non-zero `mutmut run` emits a `::warning` instead of passing silently. + **Confirmed in production** on run `30308667584` (PR #18): 461 mutants, 19 survived, 355 not covered, + 87 killed — matching the Docker measurement exactly. That run also exposed a reporting bug: `mutmut + results` lists ONLY the mutants worth looking at, so `grep -c ': killed'` is always 0 and the summary + table reported "Killed 0" on a healthy run. The count is now derived as total-minus-listed + (461 − 374 = 87), validated against that run's own artifact. + Three things are load-bearing in the mutmut 3 config, each found by a run that produced nothing: `source_paths` must be the **package** (mutmut 3 copies it into `mutants/` and runs pytest there; with a single file copied, `conftest.py` cannot import `messagefoundry.config` and every mutant returns "not diff --git a/tests/test_quality_advisory_invariants.py b/tests/test_quality_advisory_invariants.py index 2630e66d..3994b599 100644 --- a/tests/test_quality_advisory_invariants.py +++ b/tests/test_quality_advisory_invariants.py @@ -284,6 +284,21 @@ def test_mutmut_copies_the_package_not_just_the_mutated_file(code: str) -> None: assert "runner=" not in code, "mutmut 3 uses pytest_add_cli_args_test_selection" +def test_the_killed_count_is_derived_not_grepped(code: str) -> None: + """`mutmut results` lists ONLY the mutants worth looking at (survived / no tests / timeout / + suspicious). Killed mutants are never listed, so counting `': killed'` returns 0 on a perfectly + healthy run -- which is exactly what shipped in #18: CI reported `killed=0 survived=19` for a run + mutmut itself scored at 87 killed. Derive it from total-minus-listed instead. + + Verified against the real artifact from run 30308667584: 461 total - 374 listed = 87, matching + mutmut's own counter. + """ + assert "grep -c ': killed'" not in code, ( + "mutmut never lists killed mutants -- this grep always yields 0" + ) + assert "TOTAL - LISTED" in code, "the killed count must be derived from the run total" + + def test_mutmut_artifact_includes_hidden_files(workflow: dict) -> None: """.mutmut-cache is a dotfile and upload-artifact skips hidden files by default -- without this the step logs 'No files were found', uploads nothing, and still reports success."""