From 5ea3ee20a9df0bc85a14b91527781df5941e7928 Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Fri, 16 Oct 2026 14:12:51 +0200 Subject: [PATCH] feat: a mutation report on the lines a pull request changes gremlins and Stryker mutate the diff, and one pull request comment lists the mutants that survived and those no test reaches. It never fails the build. --- .coverage | Bin 53248 -> 53248 bytes .github/workflows/ci.yml | 8 ++ .github/workflows/mutation-report.yml | 99 +++++++++++++++++++++ README.md | 1 + release-please-config.json | 4 + scripts/mutation_report.py | 113 ++++++++++++++++++++++++ tests/test_mutation_report.py | 122 ++++++++++++++++++++++++++ 7 files changed, 347 insertions(+) create mode 100644 .github/workflows/mutation-report.yml create mode 100644 scripts/mutation_report.py create mode 100644 tests/test_mutation_report.py diff --git a/.coverage b/.coverage index e93a317f27310d375381e2be1f0562bcc78fed17..6c3c1e30be477fb76808497321154f41f0c3201a 100644 GIT binary patch delta 2565 zcmb8wYfM{Z7zgl{+e62DX<=h4w=rnBj%&GfKw(_UfKdvJI~~w+E0=-5G8Zr1W~R|? z8Z)m^BWk8u6P?P3X{N*>i%~O;7+uUvhS3F$nMREoH47e|^E7_;eEQ}8ym?Q~n{)o> z*h28Jh2Yo2Dnm|dKRY}#Jv*O0b99SFqtVJU8hJ+9?LHYg8*<#M+uSVo z|8BN;yFaYbrx{`68u_REN#2xS%g^OU^0ItOUXU-#=j3DZlsqAi${9H-56X7AUslUK zGEb(-oibW-86y4>zltBknz$xD5m&@JVo~_T3*ss9h`3+eBj&|~aEor?6m`NT3dL@b zEMkN~g!2vl2fxL?^xgwPqK&ENp?4zW8=!XJQ3l8kx-pZt8I3|Frs1w zh7h(2@Tgb@w+b5!A}skZfH0e&A2G852N6>X;6hAJK_6mb5_%Ei6VQY3jzc$M%nQCQ z)aV#=B8Eqy12HrV?Fi2h96-1|(1sXvLn~ro5LyreGvGwH{LqZ(bAbcV+XqdEo?h6G z=JeRj*r(3Apbp{h1UsT*0BRAAW~f25IH4NRwggp();8FSaJE3D zidLvV*d1U)IGj+9XmUVVsBiP|8|+Z3E^2^1hznL|GMNs)+?M5QW8% zjA}$^nA~p^p5HX3c1!0T>PWBJ<1+n_b83yW&Nz3H7d{a-?f;a zH%0h#VVj?UQbHr7ij`rKtzsykWlumhH_4JeNHUuSNHZJ#q^X62q{%54X=1XEG(OQw z@_EO5h-2Pv(&$(hX?V1gG&I~n@(i_;+@1rZL3bNzVX&1nFw;VE`JJRbS2L-%&#@`x zBhdErHc{B!v!B%2-AL;3H;}sg^?_;EK9aw)j?^(=CpnsHNiEJAQrl8BskN<&38~gzOsc6Z3aF`&RPC!N zATCu~NL3yAq-9$ksoa)JS}D&Vl~}S##U)v!vZ~#I%#ukeEY2WVSJO#FmNb-d{)I)U z6c!YwkSqmeQohAR$}lGfGHVhkFMk&)H!qR2n!A&fU6w#f&xjAm97oDri`9kUPe7ZR z9z$W8*+@!F+d)c6-A*#6M3YSBZ2@I&B_*4pNJ+_&q_rdiDKT45S~p5k{JKvNPXSXaFSATq{yf+l0oTM_1)4NLP$a%ObQY~;W!P}Z3Mk|O#d%v CKY1Jg literal 53248 zcmeI4dwdkt*~g!gWasSJnc2Z>q-a?w5P~7d#Rf_tfP}l;p+qimv)Pb^B%ADRNPq-A zpcR#i`dV#ME>@^gywulA#agAJRPnAA<@E+uw1`;Sw^dtf-S;_j&Pkw^e)@;~)b>65 zA<6f7=FIGw-^~0bze&i<>5~(gc%(X+s*7eKgP;o#0wW_400`lSiy!($5%P3_U&%3i zA`C%sjJVz$aIAIUEp(@P{hW<%Z~J+tj$LVQrcoBd1?FWGFbWt2i~|3^72wkyn%}i6 zS=E+_R@TNdsc0;olAqo4XOxYaRTh~wYV_o?i0noBgd+Ir*Do?El1es58se!)b)q&N zNz_**V$n>Zeo-V-6W61g((x*Bqr)EF>cqfee|!5NhIqd@s@b3DHE^y+#M_uqUjfkckJHawk{Gobdg+Bz1K17bguk4YpUUD z7e`dC)l{EoY>Iar{Gv$bnnnI@)F+D~Vkhtn=w(~^Lr)=qXJx!4-B^p?Z&gH_GD*L# zz=K~g06#gGo)o0{-Mf>^8~qbS$Wu|7$fP@f9e2`nv`=vAh~+0fZ^Tt{{7E9NqT1R> zvf967#M%*gaPVwC_L}Lh#fYZFg%(MwfjHBD>DeYGi*txn)+Q^DKiB*-CLPBM1ie5H zt8S`~iAyCd_92#B8c#(R#f#NaX$$}A6A4e90nExT>wcnBr{ff;7|?+s-qoV{-MW!g z%s-9P(NbUNai@?zlAmLvdQn|Gnr=#IBl&wKsn_UqNqvo|CAd5NsFgpe+lh8ZjjI@- zAjoMR%|Gol!sQhqd`Wx+j@t>*>AVE~>PGO7Pro6WsfqmU+(jEFUJ$VIhn;q!iHj*z z3=p5K)$?-vDPP{zSHBKTUwq^qH)YZ3obsgjvVKdWwdf`d(L@R_S@d9e2Z)G)RZ$!q ztxPs${PS0o{^=?1J)QHF8|bXH_Bg80AC8yzv}8J+z}u?0ZKc1VyODyLXj-^Xx*$@S zOxDJu_43|;YfMPvdnVEme-3|pS1Rq?MtYgp#-gPK{+#s9McDi-yy*1xg-}-)t*=t| zD{&i^TSe~|7pHh1X(G5{qyR64aCu^7$822lpGY_%JJ@m z=AUy8;Z5pdls;3D#+!OwR8L!#ImewOf3>jle3^=di{I@Wd!7jGj*&x@@*RY^$s<@P zRnVi<_d20M5%7ukXTX2vWfU+97zK<1MggOMQNSo*6fg=H1&jhl0i(cwo&vcfhftCK zbG(D#{RTgomr=kdU=%P47zK<1MggOMQNSo*6fg=H1&jj!mI^pIxo3I$H#`P(A-UbU zs^0-9DIRccaf$dP{~Yfyc!#|Y|1EdWOxh@56fg=H1&jhl0i%FXz$jo8FbWt2i~>f1 z4h5Xtvxt5SkQ2)7#?(^)`TPIP;5By)HDDAl3K#{90!9I&fKk9GU=%P47zK<1MggP1 z7oz~TNFJ;@ssHRW{*8nFi;J6!7bokN2%a#eY}B+=a&bJCN%x|f8wWF$(17x=j51AG&-o+cR4+3K#{9 z0!9I&fKk9GU=%P47zK<1MggOMQQ*s|fcW{p$^T!@V`FAz6fg=H1&jhl0i%FXz$jo8 zFbWt2i~>f1FG>OY`9Fa`9K4Uc-*|6%`@N^VN4*EUyS!Vx&E9vswccuPx!35`cop6p zZ<06K8|t0w6?k9uPV*cuH~dNXcj3d~1L2p#Plq23KM=kv{G;%V;Tyv1!L;Hw*z<$x*Yd>V)ZEv?X+1J{a+pTt^U2V^^ zr`codA$E}+u}`&wp-)4<4IK*Y4?Pw7dFYq&JFbp zoe^?E!2ZAvv)9;j>`}Ip-NtTY*ReIMl{K&`HkVCiBiUfqhn>l~u)N?OgGYl0gD(aj z5B@B8XK-8ayTP@=OM;nTO>lm2T5wEoNU$gv37#4Z(ogAc=^?tGK1F{{@1fi2MtU{9 zl(x`18l|)81Uj4!puK2U>e5{6W9vQZO>3XE*V<*>ZQWvh-@3-S+-kGZR>E3fO}EBb z!>vJ9fz{3OtlYrIf%gM%2KEK^26hGR4%`yh6u2gESztvVi6?-083l|2{~QJADhMz} z$nti^FGkAAfdkp0l&|!=%a$|VCcAII0mfT3AEbPRKVtq4#+S?P!?2t2Ws0+W?lR!T3e8yA8H6UZHs_c7Fhy884T7Bm98zxjtvx;4;R)CHt3U+Zdmt zc?ILM6^k*mB;Sy&WqhXOBXBgxX9yOMVKTX8b(KPry@* z50?BSJi+)N-G7qtfs*%%{s763!(PVE^*K8f9%sD2>_3tn&Umq6c!cqOk{=eARFNEC zD)=1Pe>6LZ@xGEDgAW)ll)M`rWxS8%N5sWe;B$7k*k*4%2Dz8yU1Cg6$q&IU#=j={ zL3oJq9*VQ)Gk&)2Kgf8*=j{1_++FtX7yMPpJK=uD&r+Nn&-j_TzmxH9K4-@R@>gX4 zKEYjOe-hls_!+W)FFeM0zT~^$F2+xnW2Oi`P4;gSe5&L-;Woxkk^E!0lkt-!-vK{n z{3OY@!ySxwk$fB6&ba4ucD7hIEc>^@y^On(w`I}l964sbxYTXgzgcid_E(5Ykg4%- zGvh(YTg4bE`6k%PxTQF|knw=e*@bWu<9V{bS#Yl88-*q0NZtZBGEVfEEsP6$N2A(= zW0b#dgiTD?drnTa8a6WJ@9l6aQ}%v6+`xpp0Kv+ADU5}`|eSJ2?l(nyi^-MZ@Hif8+eI26m^|iP$Ve7bs ztMHA5uOnUwS21DiILQ{ck|}3j1J_Yu?1Zemm`Yb~YopTCSGUnc{-V}4CM+E{vKrRV zxZhjRN~`30twG<<30%@ng_{$ytcgl1Z(B))lM~XmIw)+MYr6>d$)`V3npZ9j0*oHq_Uch@nK#$70ye@!g4BXmyn7bRCq2S z3l>sgxCrG`xGf$`f;h2Ps9Z!W_5;Crg3a`Y@I4X=1*BVQOOA<1s zj0%e+WaMZn{E?8+V`xtwMvbPz8FA4lDr}LE5v5dkBEm>243UuGBdBmgLiP-&!U_o) zHirryBxJ~5Dol`&y(LsQAR#5gsIWhTAyjxDAr0+R7#|^nOQ>)?LIw?{!tw|iIEV_r z!?UD;3bP}mcpy_wcQve|!srMoDyG8a2&rqQ!r}<&TSSGw5mMNf3Uec*Paze~Mo2*) zDr}9A-UU>68X>)UQ(2celmPtcx(9fz=81h!YjfX@VP)^co9AcSP-5No*B3)TpG9`ur_>NfQQd^ z{}j#*%y&NsOm|OAH3a(0LR;@si1I#)a2cG|7H!0k>doF8@rW8Js&K5=F_3!RD1 z*?BATs+}DBs8i~kXYY5;aUA;@tI#>!{>ZAa=2=blZhMEl*}gXKm-bieZ`+l2y*>2iOAQM=`+Sxjm#wP{ktTfP_4Gf%&N6=a#*Sgo*U~RW{TO0p5 z!N+XXC?FIFS-$NoZztd&=AcJm0EVpUFIx_u={`na_)BMT5I$9d=I?+%>prGn_@txA zmcpOZpn3D*kGhXR7(UhoCSmwU7Z`=%L$yY8JN!ZSF$}}+b%ALZ{#O?mhv9d+z&s4U z?JVAa57f4nHp6cePA!A?6;7E7NA+k7#qeHdF$IpOL6gehU4;`T!C{3H%HY=u$4`KF z6sFtZZM_Z5#&Ad%7>?mBU0^zfU#T@pN5Gr9kNFr5b{1RVfEtua!yCGf5gA_B1!iP; zO%1xZ9bVOaOv&(yE-)s;|ENObWZ18lU{HpaJBtmhM(yILo_^~3rx}QQ(a(;hWmAaIU08A0)sT%rwdHd@Dp8N zl!kkCfms^v(Zv(+B;2hFOw(|eE-+5Rf2%^|X}D7_!9Wc^?kM09xI-5hso{3L4rXe& zO)tSv4Y%q7Q#I_+1;%RlkuES-!**4OU=6qECA+b>Sr-_s;fK1wYz^CVf#Djqb`+SS zZ|W%ELHL0xM81X_^%4x&utgV`uwipYfnj=6M*%xwqbfwmhVSbon6jZ=FTt1%-_uJl zXTx`Ofk7K?(8Dlk!}WRzMs4_xE--7ub-KW?4gaMJOxtj+E--GxHM+pO4Ogo|1a8=% zmtf+C^}4{w4eRtU%-pb6FTv0aSLr2~y5UN_WE}YG25-2$qrklVZB>Zq z4VUR9n7!drU10cz)w;m+4Xbpq8MXk|1?F$K#CHvR#)b*}V#OkYuhf0a;BA^QgtzK3 zn8H`+KF09nnlXnj(~Lp9MKdPxX3ZGImns%nyh-;ljAuGHn@V>utWW9jn8zD+9|L)V zW=!Ns%^1n+HDe~PQ!GMxt?pwgU&8$REXMN1irU%|idMJPC~9q6q}Rb{Uah(-TH}gV zwpIzV&mkh6$5eA!Q>CJ|l~GNrFY@mSIRq1Wg=)4eTc~Nx0!=OR6|HHRrzq2Op`yl2 zxuR6#Tutl0r71N>(fZVEMUAOhiWV=KsVLbvLs5Nlx}wD5X^Lv=rz(nVo}y{f1$tcV zWJNWJNs8KQCaRso+&)3C5*x2*Q*4}~&9SkHDyz#B%_|?HXkq#JiYj)DRgrlPs!!xZh9`wc~NDu*hXxo3!?88b^1O)vYpqNOe8DVj2U zu%gLR1}VCrY@nj16$3OaJy(yrpueKB8O4ewPVT2@{KO(fW5=JPXk1xe-5Xb^sBCN> zMPtee6pb9+ThZt-y%dca-BT|e^)*E!N_%J;dA6eABO;3S4DYUJ*qpB_8nX8+MSDxm zR8%soo2DUOQPj}hRng#*GZYOPoUdr$pwks48%|SHJn&R~31QklMRkjcPgYddev+cT zMO_pX_VpC?DGVzr=;JEtUEnC{)!SCovsXyJ3+8^Nx;=UZ6=CvMw4~kABQg62Iyl>y zry0|Kt{R=wBPZw^1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ff669c..53b8e97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,14 @@ jobs: .venv-tests/bin/python -m coverage run --branch --source=scripts.path_buckets -m unittest tests.test_path_buckets .venv-tests/bin/python -m coverage report --show-missing --fail-under=100 + # The mutation report is what reviewers read to judge a pull request's tests, so every line and + # branch of it is held too. + - name: Run mutation report tests with branch coverage + if: ${{ !cancelled() }} + run: | + .venv-tests/bin/python -m coverage run --branch --source=scripts.mutation_report -m unittest tests.test_mutation_report + .venv-tests/bin/python -m coverage report --show-missing --fail-under=100 + # --------------------------------------------------------------------------- # Terminal aggregator. This is the single required check in the ruleset. # It runs even when an upstream job fails/cancels (if: always()) and then diff --git a/.github/workflows/mutation-report.yml b/.github/workflows/mutation-report.yml new file mode 100644 index 0000000..91950d9 --- /dev/null +++ b/.github/workflows/mutation-report.yml @@ -0,0 +1,99 @@ +name: Mutation Report + +# Reports, on a pull request, the mutants that survive on the lines it changes: a test ran the +# line, yet no test failed when it changed. gremlins mutates Go, Stryker mutates TypeScript and +# Vue. One comment is kept up to date per pull request. It is a report and never fails the build +# (workspace docs/testing-standards.md, section 6). +# +# The caller's toolchain comes from its mise.toml. A web directory needs @stryker-mutator/core and +# @stryker-mutator/vitest-runner as devDependencies, so Stryker runs with the project's own vitest. + +on: + workflow_call: + inputs: + go-directory: + description: Directory of the Go module to mutate; empty skips Go. + type: string + default: '' + web-directory: + description: Directory of the web project to mutate; empty skips TypeScript. + type: string + default: '' + gremlins-version: + description: gremlins release to install. + type: string + default: v0.6.0 + +permissions: + contents: read + pull-requests: write + +jobs: + report: + name: Mutation report + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + # Checkout github-workflows at this workflow file's own released version; release-please + # keeps the literal in step with the tag this file ships in. + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: JorisJonkers-dev/github-workflows + ref: v0.24.0 # x-release-please-version + path: .github-workflows + persist-credentials: false + + - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0 + + - name: Mutate the changed Go + if: ${{ inputs.go-directory != '' }} + continue-on-error: true + working-directory: ${{ inputs.go-directory }} + env: + BASE: origin/${{ github.base_ref }} + GREMLINS: ${{ inputs.gremlins-version }} + run: | + go install "github.com/go-gremlins/gremlins/cmd/gremlins@$GREMLINS" + "$(go env GOPATH)/bin/gremlins" unleash --diff "$BASE" --output "$RUNNER_TEMP/gremlins.json" ./... + + - name: Mutate the changed TypeScript + if: ${{ inputs.web-directory != '' }} + continue-on-error: true + working-directory: ${{ inputs.web-directory }} + env: + BASE: origin/${{ github.base_ref }} + run: | + changed=$(git diff --name-only --diff-filter=AM --relative "$BASE"...HEAD -- src \ + | grep -E '\.(ts|vue)$' | grep -Ev '\.(spec|test)\.ts$|\.gen\.ts$|/infrastructure/api/' | paste -sd, - || true) + if [ -z "$changed" ]; then + echo "No changed TypeScript or Vue source to mutate." + exit 0 + fi + pnpm install --frozen-lockfile + pnpm exec stryker run --testRunner vitest --reporters json,clear-text \ + --jsonReporter.fileName "$RUNNER_TEMP/mutation.json" --mutate "$changed" + + - name: Write the report + run: | + python3 .github-workflows/scripts/mutation_report.py \ + --gremlins "$RUNNER_TEMP/gremlins.json" --stryker "$RUNNER_TEMP/mutation.json" \ + --out "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment on the pull request + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + existing=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty') + if [ -n "$existing" ]; then + gh api --method PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$RUNNER_TEMP/comment.md" >/dev/null + else + gh api --method POST "repos/$REPO/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" >/dev/null + fi diff --git a/README.md b/README.md index f014742..ebc53af 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ should call released tags instead of branches. | `python-ci.yml` | Python Ruff, typecheck, and test workflow with overrideable commands. | | `nix-ci.yml` | Nix formatter, Statix, Deadnix, flake check, and optional validation. | | `jvm-ci.yml` | Gradle static lint/test workflow that fails Gradle warnings and deprecations. | +| `mutation-report.yml` | Mutates the lines a pull request changes (gremlins for Go, Stryker for TypeScript and Vue) and keeps one comment listing the mutants that survived. Never fails the build. | | `docker-image-ci.yml` | Build a Docker image without publishing. | | `container-publish.yml` | Publish a GHCR image with release and sha tags. | | `publish-api-clients.yml` | Generate and publish API clients from an OpenAPI spec. | diff --git a/release-please-config.json b/release-please-config.json index d31409b..5bb683e 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -54,6 +54,10 @@ "type": "generic", "path": ".github/workflows/migration-guard.yml" }, + { + "type": "generic", + "path": ".github/workflows/mutation-report.yml" + }, { "type": "generic", "path": ".github/workflows/node-ci.yml" diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py new file mode 100644 index 0000000..431e3ee --- /dev/null +++ b/scripts/mutation_report.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Turn gremlins' and Stryker's JSON reports into one pull request comment. + +Both tools run only on the lines a pull request changes. The comment lists every +mutant that survived (a test ran the line but no test failed when it changed) and, +apart, every mutant no test reaches. It is a report: it never fails the build. + + mutation_report.py --gremlins gremlins.json --stryker mutation.json --out comment.md + +A report that is absent counts as no mutants, so a repository with only Go or +only TypeScript passes one flag. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path + +# The comment is found again by this line, so a later run edits it instead of adding another. +MARKER = "" + + +@dataclass(frozen=True, order=True) +class Mutant: + file: str + line: int + mutator: str + + +def from_gremlins(report: dict) -> tuple[list[Mutant], list[Mutant]]: + """The mutants gremlins found LIVED, and those NOT COVERED.""" + survived: list[Mutant] = [] + unreached: list[Mutant] = [] + for f in report.get("files") or []: + for m in f.get("mutations") or []: + mutant = Mutant(f["file_name"], m["line"], m["type"]) + if m["status"] == "LIVED": + survived.append(mutant) + elif m["status"] == "NOT COVERED": + unreached.append(mutant) + return survived, unreached + + +def from_stryker(report: dict) -> tuple[list[Mutant], list[Mutant]]: + """The mutants Stryker marked Survived, and those with NoCoverage.""" + survived: list[Mutant] = [] + unreached: list[Mutant] = [] + for path, f in (report.get("files") or {}).items(): + for m in f.get("mutants") or []: + mutant = Mutant(path, m["location"]["start"]["line"], m["mutatorName"]) + if m["status"] == "Survived": + survived.append(mutant) + elif m["status"] == "NoCoverage": + unreached.append(mutant) + return survived, unreached + + +def _table(mutants: list[Mutant]) -> list[str]: + rows = ["| where | mutation |", "|---|---|"] + rows += [f"| `{m.file}:{m.line}` | {m.mutator} |" for m in sorted(mutants)] + return rows + + +def render(survived: list[Mutant], unreached: list[Mutant], problems: list[str] | None = None) -> str: + lines = [MARKER, "### Mutation report", ""] + if survived: + noun = "mutant" if len(survived) == 1 else "mutants" + lines += [ + f"{len(survived)} {noun} survived on the changed lines: a test ran the line, and no test failed " + "when it changed. Each is a check a test is missing.", + "", + *_table(survived), + ] + else: + lines.append("Every mutant on the changed lines was caught by a test.") + if unreached: + lines += ["", "#### No test reaches these lines", "", *_table(unreached)] + for problem in problems or []: + lines += ["", f"- {problem}"] + lines += ["", "This report never fails the build."] + return "\n".join(lines) + "\n" + + +def _read(path: str | None, name: str, problems: list[str]) -> dict: + if not path or not Path(path).is_file(): + return {} + try: + return json.loads(Path(path).read_text()) + except ValueError: + problems.append(f"The {name} report could not be read, so its mutants are not listed.") + return {} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Turn mutation reports into one pull request comment.") + parser.add_argument("--gremlins", help="gremlins' JSON report (--output)") + parser.add_argument("--stryker", help="Stryker's JSON report (json reporter)") + parser.add_argument("--out", required=True, help="where to write the comment") + args = parser.parse_args(argv) + + problems: list[str] = [] + g_survived, g_unreached = from_gremlins(_read(args.gremlins, "gremlins", problems)) + s_survived, s_unreached = from_stryker(_read(args.stryker, "Stryker", problems)) + comment = render(g_survived + s_survived, g_unreached + s_unreached, problems) + Path(args.out).write_text(comment) + print(comment) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_mutation_report.py b/tests/test_mutation_report.py new file mode 100644 index 0000000..4d258c4 --- /dev/null +++ b/tests/test_mutation_report.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import io +import json +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +from scripts.mutation_report import MARKER, Mutant, from_gremlins, from_stryker, main, render + +GREMLINS = { + "go_module": "github.com/example/app", + "files": [ + { + "file_name": "internal/access/app/service.go", + "mutations": [ + {"type": "CONDITIONALS_BOUNDARY", "status": "LIVED", "line": 42, "column": 7}, + {"type": "INVERT_NEGATIVES", "status": "KILLED", "line": 50, "column": 3}, + {"type": "ARITHMETIC_BASE", "status": "NOT COVERED", "line": 61, "column": 9}, + ], + } + ], +} + +STRYKER = { + "schemaVersion": "1", + "files": { + "src/features/home/useLauncher.ts": { + "language": "typescript", + "mutants": [ + {"id": "1", "mutatorName": "ConditionalExpression", "status": "Survived", "location": {"start": {"line": 14, "column": 5}}}, + {"id": "2", "mutatorName": "StringLiteral", "status": "Killed", "location": {"start": {"line": 15, "column": 1}}}, + {"id": "3", "mutatorName": "BooleanLiteral", "status": "NoCoverage", "location": {"start": {"line": 20, "column": 2}}}, + ], + } + }, +} + + +class ReadingGremlins(unittest.TestCase): + def test_a_mutant_that_lived_is_a_survivor(self) -> None: + survived, _ = from_gremlins(GREMLINS) + self.assertEqual(survived, [Mutant("internal/access/app/service.go", 42, "CONDITIONALS_BOUNDARY")]) + + def test_a_mutant_no_test_reaches_is_reported_apart(self) -> None: + _, unreached = from_gremlins(GREMLINS) + self.assertEqual(unreached, [Mutant("internal/access/app/service.go", 61, "ARITHMETIC_BASE")]) + + def test_a_run_with_no_changed_go_reports_nothing(self) -> None: + self.assertEqual(from_gremlins({"files": None}), ([], [])) + + +class ReadingStryker(unittest.TestCase): + def test_a_surviving_mutant_is_a_survivor(self) -> None: + survived, _ = from_stryker(STRYKER) + self.assertEqual(survived, [Mutant("src/features/home/useLauncher.ts", 14, "ConditionalExpression")]) + + def test_a_mutant_without_coverage_is_reported_apart(self) -> None: + _, unreached = from_stryker(STRYKER) + self.assertEqual(unreached, [Mutant("src/features/home/useLauncher.ts", 20, "BooleanLiteral")]) + + +class Rendering(unittest.TestCase): + def test_the_comment_carries_the_marker_it_is_found_by(self) -> None: + self.assertTrue(render([], []).startswith(MARKER)) + + def test_no_survivors_says_every_mutant_was_caught(self) -> None: + self.assertIn("Every mutant on the changed lines was caught by a test.", render([], [])) + + def test_each_survivor_is_listed_with_where_it_lives(self) -> None: + text = render([Mutant("a.go", 3, "INVERT_NEGATIVES")], []) + self.assertIn("| `a.go:3` | INVERT_NEGATIVES |", text) + self.assertIn("1 mutant survived", text) + + def test_unreached_mutants_get_their_own_section(self) -> None: + text = render([], [Mutant("b.ts", 9, "StringLiteral")]) + self.assertIn("No test reaches these lines", text) + self.assertIn("| `b.ts:9` | StringLiteral |", text) + + def test_the_comment_says_it_never_fails_the_build(self) -> None: + self.assertIn("This report never fails the build.", render([], [])) + + def test_survivors_are_sorted_by_file_and_line(self) -> None: + text = render([Mutant("b.go", 1, "X"), Mutant("a.go", 9, "Y"), Mutant("a.go", 2, "Z")], []) + self.assertLess(text.index("`a.go:2`"), text.index("`a.go:9`")) + self.assertLess(text.index("`a.go:9`"), text.index("`b.go:1`")) + + +class CommandLine(unittest.TestCase): + def test_it_writes_one_comment_from_both_reports(self) -> None: + with tempfile.TemporaryDirectory() as directory: + gremlins, stryker, out = (Path(directory) / name for name in ("g.json", "s.json", "comment.md")) + gremlins.write_text(json.dumps(GREMLINS)) + stryker.write_text(json.dumps(STRYKER)) + with redirect_stdout(io.StringIO()): + code = main(["--gremlins", str(gremlins), "--stryker", str(stryker), "--out", str(out)]) + text = out.read_text() + self.assertEqual(code, 0) + self.assertIn("`internal/access/app/service.go:42`", text) + self.assertIn("`src/features/home/useLauncher.ts:14`", text) + + def test_a_missing_report_counts_as_no_mutants(self) -> None: + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) / "comment.md" + with redirect_stdout(io.StringIO()): + code = main(["--gremlins", str(Path(directory) / "absent.json"), "--out", str(out)]) + self.assertEqual(code, 0) + self.assertIn("Every mutant on the changed lines was caught by a test.", out.read_text()) + + def test_a_report_that_is_not_json_is_named_but_does_not_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + broken, out = Path(directory) / "g.json", Path(directory) / "comment.md" + broken.write_text("not json") + with redirect_stdout(io.StringIO()): + code = main(["--gremlins", str(broken), "--out", str(out)]) + self.assertEqual(code, 0) + self.assertIn("could not be read", out.read_text()) + + +if __name__ == "__main__": + unittest.main()