Repository navigation
145 lines (127 loc) · 5.94 KB
/
Copy pathmutation-test.yml
File metadata and controls
145 lines (127 loc) · 5.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
name: "Mutation Test (manual)"
# Manually-triggered mutation testing. Runs mutmut against the scope
# configured in [tool.mutmut] in pyproject.toml (currently the
# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly
# weekly — clicked from the Actions tab when someone wants a fresh report.
#
# Uploads a structured `mutation-report.md` (Meta ACH-style: original +
# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the
# existing tests + a task instruction) as a workflow artifact. Failures
# do not block anything because nothing depends on this workflow.
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: mutation-test-${{ github.ref }}
cancel-in-progress: true
jobs:
mutation:
name: Run mutmut
runs-on: ubuntu-latest
# Whole-folder mutation against ~15 files / ~7.5k LOC can take hours.
# 350 minutes is just under the GitHub-hosted job cap of 360 minutes.
timeout-minutes: 350
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
# mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
# mutation trampolines. uv installs the project as editable by default,
# which puts the original source dir on sys.path via a .pth file and
# shadows the sandbox copy — so tests would never exercise the mutated
# code. Reinstalling non-editable removes the .pth shadow.
- name: Reinstall litellm non-editable (so mutants/ is not shadowed)
run: |
uv pip uninstall litellm
uv pip install . --no-deps
# pytest-retry's pytest_configure hook crashes with
# `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
# via mutmut's in-process pytest.main() call. The entry-point name
# doesn't normalize cleanly with `-p no:<name>`, so just remove the
# package outright. Reruns are wrong for mutation testing anyway —
# rerunning a "failed" mutant test would mask which mutants are killed.
- name: Remove pytest plugins that conflict with mutmut
run: |
uv pip uninstall pytest-retry || true
# Ends before the job's own deadline so a run that outlasts the budget is
# still followed by the report and upload steps. mutmut saves after every
# mutant result, to mutants/<source path>.meta, so an interrupted run
# still scores the mutants it finished and export-cicd-stats can read
# them; a cancelled job skips those steps and publishes nothing at all.
- name: Run mutmut
timeout-minutes: 300
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
# Without this mutmut finds no covered lines and generates 0 mutants.
# See the file itself for why.
COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
run: |
set -o pipefail
mkdir -p mutants
uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log
# Generate the structured report. The script embeds the enclosing
# function source for each survivor (via Python AST) and includes the
# existing test files, so an LLM agent has enough context to write
# killing tests without further file lookups. Modeled on Meta's ACH
# prompt template (arXiv 2501.12862).
- name: Generate detailed mutation report
if: always()
run: |
set +e
uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
uv run --no-sync python scripts/mutation_report.py
# The full report can be very long for big test files; the run-page
# summary cuts off at 1 MB. Append the head of the report (summary
# + survivor list) and link out to the artifact for the full body.
{
head -c 900000 mutation-report.md
echo ""
echo ""
echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload mutmut artifacts
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
path: |
mutation-report.md
mutmut-results.txt
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/**/*.meta
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14