Skip to content

Commit e89dc85

Browse files
committed
feat(coverage): decide the touched-function CRAP verdict
`check:coverage-crap --base <ref>` was report-only: a PR that pushed a function over the threshold printed a warning annotation and went green. It now returns 1 for that PR, and also when the reports cannot be trusted to answer at all — a missing, malformed, or partial coverage map, a touched file the map dropped or never recorded, or a touched function the map predates, which fallow scores from complexity alone. `--all` stays the advisory whole-repo backlog that `main` reports. Attribution reuses the rename-aware diff, so moving a file owes nothing for the functions that travelled with it, while a rename that also edits a body still owes those lines. lcov is read as the same coverage run's own list of measured files, which is what makes a touched file absent from the map a named gap instead of a clean result. The report now also lists the file-wide `fallow-crap.json` exemptions it honoured, read from the resolved config rather than a copy that can drift. CI keeps its report-only step: flipping it is the next change, so this one can be reviewed as the verdict alone. Part of #3426
1 parent 18d22c7 commit e89dc85

3 files changed

Lines changed: 763 additions & 73 deletions

File tree

‎scripts/coverage-crap/model.test.ts‎

Lines changed: 234 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,18 @@ import assert from 'node:assert/strict';
22
import fs from 'node:fs';
33
import { test } from 'node:test';
44
import {
5+
coverageMapPaths,
6+
decideCrapVerdict,
57
parseFallowJsonReport,
8+
parseIstanbulCoverageMap,
9+
readCrapExemptions,
610
readCrapFindings,
711
renderCrapAnnotations,
812
renderCrapReport,
913
sanitizeIstanbulCoverage,
14+
type CrapFinding,
15+
type CrapScope,
16+
type CrapVerdict,
1017
} from './model.ts';
1118

1219
const valid = { s: { 0: 3, 1: 0 }, f: { 0: 1 }, b: { 0: [2, 0] } };
@@ -71,13 +78,21 @@ test('a report without a findings array is an error, not an empty verdict', () =
7178
assert.throws(() => readCrapFindings({ error: true }, 30), /no findings array/);
7279
});
7380

81+
/** A verdict that owes nothing: what `decideCrapVerdict` returns when the reports answer cleanly. */
82+
const clean = (scope: CrapScope, findings: readonly CrapFinding[] = []): CrapVerdict => ({
83+
blocking: false,
84+
problems: [],
85+
offenders: scope === 'touched' ? findings : [],
86+
});
87+
7488
test('the report names touched functions and unscored files', () => {
7589
const findings = readCrapFindings({ findings: [finding({})] }, 30);
7690
const report = renderCrapReport({
7791
findings,
7892
dropped: ['src/odd.ts'],
7993
threshold: 30,
8094
scope: 'touched',
95+
verdict: { blocking: true, problems: [], offenders: findings },
8196
});
8297
assert.match(report, /1 touched function\(s\) at or above 30/);
8398
assert.match(report, /\| `src\/a\.ts:10` run \| 42\.0 \| 6 \| 20% \|/);
@@ -88,17 +103,234 @@ test('the report names touched functions and unscored files', () => {
88103
});
89104

90105
test('an empty report says none and lists no table', () => {
91-
const report = renderCrapReport({ findings: [], dropped: [], threshold: 30, scope: 'touched' });
106+
const report = renderCrapReport({
107+
findings: [],
108+
dropped: [],
109+
threshold: 30,
110+
scope: 'touched',
111+
verdict: clean('touched'),
112+
});
92113
assert.match(report, /: no touched function/);
93114
assert.doesNotMatch(report, /\| Function/);
115+
// Each section owns its own emptiness guard, so a clean run must not announce empty ones: an
116+
// "Unscored: 0 file(s)" or "exemptions (0)" line reads as a finding to someone scanning the log.
117+
assert.doesNotMatch(report, /Unscored:/);
118+
assert.doesNotMatch(report, /exemptions/i);
94119
});
95120

96121
test('a whole-repository report does not claim the functions were touched', () => {
97122
const findings = readCrapFindings({ findings: [finding({})] }, 30);
98-
const report = renderCrapReport({ findings, dropped: [], threshold: 30, scope: 'all' });
123+
const report = renderCrapReport({
124+
findings,
125+
dropped: [],
126+
threshold: 30,
127+
scope: 'all',
128+
verdict: clean('all'),
129+
});
99130
assert.match(report, /: 1 function\(s\) at or above 30/);
100131
});
101132

133+
test('a touched report whose reports cannot answer names the reason and stays ahead of the table', () => {
134+
const report = renderCrapReport({
135+
findings: [],
136+
dropped: [],
137+
threshold: 30,
138+
scope: 'touched',
139+
verdict: {
140+
blocking: true,
141+
offenders: [],
142+
problems: [
143+
{ kind: 'unscored_touched', files: [{ path: 'src/new.ts', addedLines: 4 }] },
144+
{ kind: 'corrupt_touched', files: ['src/odd.ts'] },
145+
],
146+
},
147+
});
148+
assert.match(report, /no trustworthy verdict/);
149+
assert.match(report, /no record in the coverage map\*\*: `src\/new\.ts` \(4 changed line\(s\)\)/);
150+
assert.match(report, /impossible counts\*\*: `src\/odd\.ts`/);
151+
// Every problem carries the command that clears it, since the reader has no other way to act.
152+
assert.match(report, /Run `pnpm test:coverage` again/);
153+
});
154+
155+
test('an unanswerable touched report never reads as the empty verdict it was computed from', () => {
156+
const report = renderCrapReport({
157+
findings: [],
158+
dropped: [],
159+
threshold: 30,
160+
scope: 'touched',
161+
verdict: {
162+
blocking: true,
163+
offenders: [],
164+
problems: [{ kind: 'enumeration_missing', detail: 'no coverage/lcov.info' }],
165+
},
166+
});
167+
assert.doesNotMatch(report, /no touched function\(s\)/);
168+
assert.match(
169+
report,
170+
/which files the coverage run enumerated is missing or unreadable\*\* \(no coverage\/lcov\.info\)/,
171+
);
172+
});
173+
174+
test('the same gap in the whole-repo report is a note, not a verdict', () => {
175+
const report = renderCrapReport({
176+
findings: [],
177+
dropped: [],
178+
threshold: 30,
179+
scope: 'all',
180+
verdict: {
181+
blocking: false,
182+
offenders: [],
183+
problems: [{ kind: 'map_partial', mapFiles: 9, enumeratedFiles: 12 }],
184+
},
185+
});
186+
assert.match(report, /report-only\): no function\(s\) at or above 30/);
187+
assert.match(report, /Not gating, but the report is incomplete/);
188+
assert.match(report, /\(9 recorded, 12 enumerated\)/);
189+
});
190+
191+
test('the exemptions are read from the resolved config and stated with their file-wide scope', () => {
192+
const exemptions = readCrapExemptions({
193+
health: {
194+
thresholdOverrides: [
195+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
196+
{ files: ['src/cheaper.ts'], maxCrap: 12, reason: 'A cap, not an exemption.' },
197+
{ files: ['src/no-crap.ts'], reason: 'Raises nothing.' },
198+
],
199+
},
200+
});
201+
assert.deepEqual(exemptions, [
202+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
203+
]);
204+
const report = renderCrapReport({
205+
findings: [],
206+
dropped: [],
207+
threshold: 30,
208+
scope: 'all',
209+
verdict: clean('all'),
210+
exemptions: { available: true, exemptions },
211+
});
212+
assert.match(report, /exemptions in `fallow-crap\.json` \(1\).*every function in its files/s);
213+
assert.match(report, /- `src\/worker\.ts` — `maxCrap: 1e\+12`: Worker thread\./);
214+
});
215+
216+
test('exemptions that could not be read are reported as unknown rather than as none', () => {
217+
assert.deepEqual(readCrapExemptions({ health: {} }), []);
218+
const report = renderCrapReport({
219+
findings: [],
220+
dropped: [],
221+
threshold: 30,
222+
scope: 'all',
223+
verdict: clean('all'),
224+
exemptions: { available: false, detail: 'fallow config exited with code 1' },
225+
});
226+
assert.match(report, /could not be read from the config \(fallow config exited with code 1\)/);
227+
assert.doesNotMatch(report, /exemptions in `fallow-crap\.json` \(0\)/);
228+
});
229+
230+
test('the verdict blocks a touched function at or above the threshold', () => {
231+
const findings = readCrapFindings({ findings: [finding({})] }, 30);
232+
const verdict = decideCrapVerdict({
233+
scope: 'touched',
234+
findings,
235+
estimated: [],
236+
touched: [{ path: 'src/a.ts', addedLines: 2 }],
237+
dropped: new Set(),
238+
mapPaths: new Set(['src/a.ts']),
239+
enumeratedPaths: new Set(['src/a.ts']),
240+
ioProblems: [],
241+
});
242+
assert.equal(verdict.blocking, true);
243+
assert.deepEqual(verdict.problems, []);
244+
assert.deepEqual(verdict.offenders, findings);
245+
});
246+
247+
test('the whole-repo verdict reports the backlog without blocking', () => {
248+
const findings = readCrapFindings({ findings: [finding({})] }, 30);
249+
const verdict = decideCrapVerdict({
250+
scope: 'all',
251+
findings,
252+
estimated: [],
253+
touched: [],
254+
dropped: new Set(),
255+
mapPaths: new Set(['src/a.ts']),
256+
enumeratedPaths: new Set(['src/a.ts']),
257+
ioProblems: [],
258+
});
259+
assert.equal(verdict.blocking, false);
260+
assert.deepEqual(verdict.offenders, []);
261+
});
262+
263+
test('a touched file the sanitizer dropped is named once, as corrupt', () => {
264+
const verdict = decideCrapVerdict({
265+
scope: 'touched',
266+
findings: [],
267+
estimated: [],
268+
touched: [{ path: 'src/odd.ts', addedLines: 1 }],
269+
dropped: new Set(['src/odd.ts']),
270+
mapPaths: new Set(['src/odd.ts']),
271+
enumeratedPaths: new Set(['src/odd.ts']),
272+
ioProblems: [],
273+
});
274+
assert.deepEqual(
275+
verdict.problems.map((problem) => problem.kind),
276+
['corrupt_touched'],
277+
);
278+
});
279+
280+
test('an unmeasured touched function is a problem only in a file the map records', () => {
281+
const estimated = [{ path: 'src/a.ts', name: 'added', line: 40 }];
282+
const decide = (mapPaths: string[]) =>
283+
decideCrapVerdict({
284+
scope: 'touched',
285+
findings: [],
286+
estimated,
287+
touched: [{ path: 'src/a.ts', addedLines: 1 }],
288+
dropped: new Set(),
289+
mapPaths: new Set(mapPaths),
290+
enumeratedPaths: new Set(['src/a.ts']),
291+
ioProblems: [],
292+
}).problems.map((problem) => problem.kind);
293+
// The map has the file but not the function: it predates the code and the score is a guess.
294+
assert.deepEqual(decide(['src/a.ts']), ['unmeasured_touched']);
295+
// A file entirely outside the map is already reported as unscored, not also as unmeasured — and
296+
// an empty map is partial on its own, so both names appear and neither is a duplicate.
297+
assert.deepEqual(decide([]), ['map_partial', 'unscored_touched']);
298+
});
299+
300+
test('a map that records every enumerated file is trusted, and none is a partial map', () => {
301+
const decide = (mapFiles: string[], enumerated: string[]) =>
302+
decideCrapVerdict({
303+
scope: 'touched',
304+
findings: [],
305+
estimated: [],
306+
touched: [{ path: 'src/a.ts', addedLines: 1 }],
307+
dropped: new Set(),
308+
mapPaths: new Set(mapFiles),
309+
enumeratedPaths: new Set(enumerated),
310+
ioProblems: [],
311+
}).problems.map((problem) => problem.kind);
312+
assert.deepEqual(decide(['src/a.ts', 'src/b.ts'], ['src/a.ts', 'src/b.ts']), []);
313+
assert.deepEqual(decide(['src/a.ts'], ['src/a.ts', 'src/b.ts']), ['map_partial']);
314+
});
315+
316+
test('a coverage map is only readable as files to counter objects', () => {
317+
assert.deepEqual(parseIstanbulCoverageMap('{"src/a.ts":{"s":{"0":0}}}'), {
318+
'src/a.ts': { s: { 0: 0 } },
319+
});
320+
assert.throws(() => parseIstanbulCoverageMap('not json'), /not valid JSON/);
321+
assert.throws(() => parseIstanbulCoverageMap('[]'), /an array, not a file -> counters object/);
322+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":[]}'), /for src\/a\.ts is object/);
323+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":{"s":[]}}'), /has s as object/);
324+
});
325+
326+
test("coverage keys from another checkout do not match this root's files", () => {
327+
const paths = coverageMapPaths(
328+
{ '/work/other/src/a.ts': {}, 'src/relative.ts': {} },
329+
'/work/this',
330+
);
331+
assert.deepEqual([...paths], ['../other/src/a.ts', 'src/relative.ts']);
332+
});
333+
102334
test('annotations put each finding on its file and line', () => {
103335
const findings = readCrapFindings({ findings: [finding({})] }, 30);
104336
assert.equal(

0 commit comments

Comments
 (0)