From d22b2382e5d7b2e8d83891564b187bba5bf7518e Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 22:51:37 -0500 Subject: [PATCH 1/8] Diff: run the changed test files together for their coverage The mutation layer found what each changed test file covers by running Vitest on that file alone, at the head and again at the merge base: one process, one start-up and one pass over the coverage include for each file, one after another. The functions it needs are those any of the files ran, so the head versions now run in one run, and the merge-base copies in another; a group that gives nothing (one of its files fails, or the run leaves no report) runs file by file as before, so the file that cannot run is found and named exactly as it was. At most 100 files run at once, so a very large diff never overruns a command line. The functions in scope, the mutate ranges and the verdicts are the same; coveredFunctions are now ordered by file and line. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/batch.test.ts | 219 +++++++++++++++++++++++++++++++++ src/diff/coverage-many.test.ts | 53 ++++++++ src/diff/coverage.ts | 28 +++-- src/diff/real-batch.test.ts | 100 +++++++++++++++ src/diff/scope.ts | 211 +++++++++++++++++++++++-------- 5 files changed, 555 insertions(+), 56 deletions(-) create mode 100644 src/diff/batch.test.ts create mode 100644 src/diff/coverage-many.test.ts create mode 100644 src/diff/real-batch.test.ts diff --git a/src/diff/batch.test.ts b/src/diff/batch.test.ts new file mode 100644 index 0000000..fe5ab93 --- /dev/null +++ b/src/diff/batch.test.ts @@ -0,0 +1,219 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { realExec } from "../gate/exec.ts"; +import type { Exec } from "../gate/types.ts"; +import type { CoverageOf, CoverageOfMany } from "./scope.ts"; +import { diffScope } from "./scope.ts"; +import { cleanRepos, commit, git, makeRepo, write } from "./test-support.ts"; +import type { CoverageMap, ScopeResult } from "./types.ts"; + +// The changed test files are run together, once, and file by file only where that gives nothing: the functions in +// scope, the mutate ranges and the verdicts are those of a run of each file alone. + +afterEach(cleanRepos); + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "", +].join("\n"); +const STRONG = 'import { f, g } from "./m.ts";\nf(1);\ng(2);\n'; +const WEAK = 'import { f } from "./m.ts";\nf(1);\n'; + +/** What a run covered of src/m.ts: f at lines 1 to 3, g at 4 to 6. */ +const ran = (f: boolean, g: boolean): CoverageMap => + new Map([ + [ + "src/m.ts", + [ + { start: 1, end: 3, hits: f ? 1 : 0 }, + { start: 4, end: 6, hits: g ? 1 : 0 }, + ], + ], + ]); + +function added(...names: string[]): string { + const dir = makeRepo({ "src/m.ts": M }); + write(dir, Object.fromEntries(names.map((name) => [name, WEAK]))); + git(dir, "add", "-A"); + return dir; +} + +function weakened(): string { + const dir = makeRepo({ "src/m.ts": M, "src/m.test.ts": STRONG }); + git(dir, "switch", "-q", "-c", "weaken"); + write(dir, { "src/m.test.ts": WEAK, "src/n.test.ts": WEAK }); + commit(dir, "weaken one test and add another"); + return dir; +} + +const keys = (result: ScopeResult): string[] => + result.outcome === "fail" ? [] : result.scope.coveredFunctions.map((ref) => ref.key); + +describe("changed test files run together", () => { + it("runs the head versions in one run, and none of them alone", async () => { + const together: (readonly string[])[] = []; + const alone: string[] = []; + const result = await diffScope({ + cwd: added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + base: "main", + coverageOfMany: (group) => { + together.push(group); + return Promise.resolve(ran(true, true)); + }, + coverageOf: (file) => { + alone.push(file); + return Promise.resolve(undefined); + }, + }); + expect(together).toEqual([["src/a.test.ts", "src/b.test.ts", "src/c.test.ts"]]); + expect(alone).toEqual([]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + + it("runs them one by one, in order, when the run together gives nothing, and adds up what each covered", async () => { + const alone: string[] = []; + const result = await diffScope({ + cwd: added("src/a.test.ts", "src/b.test.ts"), + base: "main", + coverageOfMany: () => Promise.resolve(undefined), + coverageOf: (file) => { + alone.push(file); + return Promise.resolve(file === "src/a.test.ts" ? ran(true, false) : ran(false, true)); + }, + }); + expect(alone).toEqual(["src/a.test.ts", "src/b.test.ts"]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + + it("fails naming the first file that cannot run alone, the later ones unnamed", async () => { + const result = await diffScope({ + cwd: added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + base: "main", + coverageOfMany: () => Promise.resolve(undefined), + coverageOf: (file) => Promise.resolve(file === "src/a.test.ts" ? ran(true, true) : undefined), + }); + expect(result).toEqual({ + outcome: "fail", + reason: "could not run src/b.test.ts to find the functions it covers", + }); + }); + + it("is the same scope as running each file alone", async () => { + const dir = added("src/a.test.ts", "src/b.test.ts"); + const maps: Record = { + "src/a.test.ts": ran(true, false), + "src/b.test.ts": ran(false, true), + }; + const alone: CoverageOf = (file) => Promise.resolve(maps[file]); + const many: CoverageOfMany = () => Promise.resolve(ran(true, true)); + const one = await diffScope({ cwd: dir, base: "main", coverageOf: alone }); + const all = await diffScope({ + cwd: dir, + base: "main", + coverageOfMany: many, + coverageOf: alone, + }); + expect(all).toEqual(one); + }); + + it("runs the merge-base versions together too, as copies that are gone afterwards", async () => { + const dir = weakened(); + const together: (readonly string[])[] = []; + const seen: string[] = []; + const result = await diffScope({ + copyToken: () => "tok", + cwd: dir, + base: "main", + coverageOf: () => Promise.resolve(undefined), + coverageOfMany: (group) => { + together.push(group); + for (const file of group) seen.push(existsSync(join(dir, file)) ? file : ""); + return Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? ran(true, true) : ran(true, false), + ); + }, + }); + expect(together).toEqual([["src/m.test.ts", "src/n.test.ts"], ["src/m.wt-base-tok.test.ts"]]); + expect(seen).toEqual(["src/m.test.ts", "src/n.test.ts", "src/m.wt-base-tok.test.ts"]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("holds the merge-base version in the copy, and runs a copy that fails together alone, leaving it out", async () => { + const dir = weakened(); + const alone: string[] = []; + const text: Record = {}; + const result = await diffScope({ + copyToken: () => "tok", + cwd: dir, + base: "main", + coverageOfMany: (group) => + Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? undefined : ran(true, false), + ), + coverageOf: (file) => { + alone.push(file); + text[file] = readFileSync(join(dir, file), "utf8"); + return Promise.resolve(undefined); + }, + }); + expect(alone).toEqual(["src/m.wt-base-tok.test.ts"]); + expect(text["src/m.wt-base-tok.test.ts"]).toBe(STRONG.trim()); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("leaves a merge-base version out when its runs throw, and still removes the copy", async () => { + const dir = weakened(); + const result = await diffScope({ + copyToken: () => "tok", + cwd: dir, + base: "main", + coverageOfMany: (group) => { + if (group.some((file) => file.includes(".wt-base-"))) throw new Error("the run died"); + return Promise.resolve(ran(true, false)); + }, + coverageOf: () => { + throw new Error("the run died"); + }, + }); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("runs at most 100 files at once", async () => { + const names = Array.from({ length: 205 }, (_, at) => `src/t${String(at)}.test.ts`); + const sizes: number[] = []; + await diffScope({ + cwd: added(...names), + base: "main", + coverageOfMany: (group) => { + sizes.push(group.length); + return Promise.resolve(ran(true, false)); + }, + }); + expect(sizes).toEqual([100, 100, 5]); + }); + + it("runs no group when it is given a source for one file only", async () => { + const asked: string[] = []; + const exec: Exec = (command, args, options) => { + asked.push(command); + return realExec(command, args, options); + }; + const result = await diffScope({ + cwd: added("src/a.test.ts", "src/b.test.ts"), + base: "main", + exec, + coverageOf: () => Promise.resolve(ran(true, false)), + }); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(asked.filter((command) => command.includes("vitest"))).toEqual([]); + }); +}); diff --git a/src/diff/coverage-many.test.ts b/src/diff/coverage-many.test.ts new file mode 100644 index 0000000..8974952 --- /dev/null +++ b/src/diff/coverage-many.test.ts @@ -0,0 +1,53 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { Exec } from "../gate/types.ts"; +import { vitestCoverageOf, vitestCoverageOfMany } from "./coverage.ts"; + +const REPORT = { + "/repo/src/a.ts": { + fnMap: { 0: { loc: { start: { line: 1 }, end: { line: 3 } } } }, + f: { 0: 1 }, + }, +}; + +/** An exec that writes the report for a run that exits 0, and keeps the arguments of every run. */ +function stamped(log: string[][], code = 0): Exec { + return (command, args) => { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")); + const out = flag?.slice("--coverage.reportsDirectory=".length) ?? ""; + mkdirSync(out, { recursive: true }); + writeFileSync(join(out, "coverage-final.json"), JSON.stringify(REPORT)); + log.push([command, ...args]); + return Promise.resolve({ code, stdout: "", stderr: "" }); + }; +} + +describe("vitestCoverageOfMany", () => { + it("runs Vitest once on every file it is given, and reads one map", async () => { + const log: string[][] = []; + const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped(log), + }); + expect(log).toHaveLength(1); + const args = log[0] ?? []; + expect(args.slice(1, 4)).toEqual(["run", "src/a.test.ts", "src/b.test.ts"]); + expect(map?.get("src/a.ts")).toEqual([{ start: 1, end: 3, hits: 1 }]); + }); + + it("is undefined when the run fails, whichever file is the cause", async () => { + const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped([], 1), + }); + expect(map).toBeUndefined(); + }); + + it("is, for one file, the run of that file alone", async () => { + const log: string[][] = []; + await vitestCoverageOf("src/a.test.ts", { cwd: "/repo", exec: stamped(log) }); + expect((log[0] ?? []).slice(1, 3)).toEqual(["run", "src/a.test.ts"]); + expect((log[0] ?? []).filter((arg) => arg.endsWith(".test.ts"))).toEqual(["src/a.test.ts"]); + }); +}); diff --git a/src/diff/coverage.ts b/src/diff/coverage.ts index 0bdb89f..d894349 100644 --- a/src/diff/coverage.ts +++ b/src/diff/coverage.ts @@ -101,11 +101,16 @@ export interface CoverageRun { readonly args?: readonly string[]; } -function coverageArgs(testFile: string, config: string, out: string, run: CoverageRun): string[] { +function coverageArgs( + testFiles: readonly string[], + config: string, + out: string, + run: CoverageRun, +): string[] { const excludes = [...DEFAULT_COVERAGE_EXCLUDES, ...(run.exclude ?? [])]; return [ "run", - testFile, + ...testFiles, `--config=${config}`, "--coverage", "--coverage.provider=v8", @@ -118,18 +123,19 @@ function coverageArgs(testFile: string, config: string, out: string, run: Covera } /** - * Runs Vitest on one test file with the gate's own config (the project's, without coverage thresholds) and - * the gate's coverage include and exclude, as the tests layer does, and reads what ran from the JSON report. - * A run that fails, or leaves no coverage file, is undefined: what the test covers is then not known. + * Runs Vitest on test files, all in one run, with the gate's own config (the project's, without coverage thresholds) + * and the gate's coverage include and exclude, as the tests layer does, and reads what ran from the JSON report: a + * function is covered when any of the files ran it. A run that fails, or leaves no coverage file, is undefined: what + * the files cover is then not known, and which of them is the cause is not either. */ -export async function vitestCoverageOf( - testFile: string, +export async function vitestCoverageOfMany( + testFiles: readonly string[], run: CoverageRun, ): Promise { const out = mkdtempSync(join(tmpdir(), "wt-cov-")); try { const config = writeKitConfig(run.cwd, out); - const args = coverageArgs(testFile, config, out, run); + const args = coverageArgs(testFiles, config, out, run); const result = await run.exec(join(run.cwd, "node_modules/.bin/vitest"), args, { cwd: run.cwd, }); @@ -139,6 +145,12 @@ export async function vitestCoverageOf( } } +/** What one test file covers: a run of that file alone. */ +export const vitestCoverageOf = ( + testFile: string, + run: CoverageRun, +): Promise => vitestCoverageOfMany([testFile], run); + /** The work tree under its given name and under its real path, when it has one. */ function rootsOf(root: string): readonly string[] { try { diff --git a/src/diff/real-batch.test.ts b/src/diff/real-batch.test.ts new file mode 100644 index 0000000..b5e34ce --- /dev/null +++ b/src/diff/real-batch.test.ts @@ -0,0 +1,100 @@ +import { rmSync } from "node:fs"; +import { afterEach, describe, expect, it } from "vitest"; +import { realExec } from "../gate/exec.ts"; +import { REAL_TOOL_TIMEOUT } from "../gate/test-support-timeouts.ts"; +import { realProject } from "../layers/tests/testkit.ts"; +import { vitestCoverageOf } from "./coverage.ts"; +import { diffScope } from "./scope.ts"; +import { commit, git, write } from "./test-support.ts"; + +// With the real Vitest: the changed test files run together give the scope that a run of each alone gives, and the +// same verdict when one of them cannot run. + +const dirs: string[] = []; + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "export function h(a: number): number {", + " return a - 1;", + "}", + "", +].join("\n"); +const test = (calls: string): string => + `import { expect, it } from "vitest";\nimport { f, g, h } from "./m.ts";\nit("runs", () => {\n expect(${calls}).toBeTypeOf("number");\n});\n`; + +function branch(files: Readonly>): string { + const { project } = realProject({ ".gitignore": "node_modules\n.web-tops\n", ...files }); + dirs.push(project.dir); + git(project.dir, "init", "-q", "-b", "main"); + commit(project.dir, "base"); + git(project.dir, "switch", "-q", "-c", "work"); + return project.dir; +} + +describe("the real run of changed test files together", () => { + it( + "is the scope of a run of each alone: added, weakened and unchanged-source tests", + async () => { + const cwd = branch({ + "src/m.ts": M, + "src/weak.test.ts": test("f(1) + g(1)"), + }); + write(cwd, { + "src/weak.test.ts": test("f(1)"), + "src/a.test.ts": test("g(2)"), + "src/b.test.ts": test("f(3)"), + }); + commit(cwd, "work"); + const alone = (file: string) => vitestCoverageOf(file, { cwd, exec: realExec }); + const before = await diffScope({ + copyToken: () => "tok", + cwd, + base: "main", + coverageOf: alone, + }); + const after = await diffScope({ copyToken: () => "tok", cwd, base: "main" }); + expect(after).toEqual(before); + expect(after.outcome).toBe("scope"); + if (after.outcome !== "scope") return; + expect(after.scope.coveredFunctions.map((ref) => ref.key)).toEqual([ + "src/m.ts::f", + "src/m.ts::g", + ]); + expect(after.scope.mutate).toEqual(["src/m.ts:1-6"]); + }, + REAL_TOOL_TIMEOUT * 3, + ); + + it( + "fails as a run of each alone fails, naming the first file that cannot run", + async () => { + const cwd = branch({ "src/m.ts": M }); + write(cwd, { + "src/a.test.ts": test("f(1)"), + "src/b.test.ts": + 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n', + "src/c.test.ts": + 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n', + }); + commit(cwd, "work"); + const alone = (file: string) => vitestCoverageOf(file, { cwd, exec: realExec }); + const before = await diffScope({ cwd, base: "main", coverageOf: alone }); + const after = await diffScope({ cwd, base: "main" }); + expect(after).toEqual(before); + expect(after).toEqual({ + outcome: "fail", + reason: "could not run src/b.test.ts to find the functions it covers", + }); + }, + REAL_TOOL_TIMEOUT * 3, + ); +}); diff --git a/src/diff/scope.ts b/src/diff/scope.ts index ec4f4f8..981a0c9 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -3,7 +3,12 @@ import { readFileSync, rmSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import { realExec } from "../gate/exec.ts"; import type { Exec } from "../gate/types.ts"; -import { vitestCoverageOf, coveredSpans } from "./coverage.ts"; +import { + vitestCoverageOf, + vitestCoverageOfMany, + coveredSpans, + type CoverageRun, +} from "./coverage.ts"; import { functionsIn, innermostAt } from "./functions.ts"; import { diffText, mergeBaseOf, showAt } from "./git.ts"; import { parseDiff } from "./parse.ts"; @@ -18,6 +23,9 @@ import type { export type CoverageOf = (testFile: string) => Promise; +/** What several test files cover together: one run for all of them, the functions any of them ran. */ +export type CoverageOfMany = (testFiles: readonly string[]) => Promise; + export interface ScopeOptions { readonly cwd: string; /** The ref the run is judged against, such as `origin/main`. */ @@ -25,6 +33,11 @@ export interface ScopeOptions { readonly exec?: Exec; /** What a test file covers; by default a Vitest run of that file alone. */ readonly coverageOf?: CoverageOf; + /** + * What test files cover together, in one run: used, when given, before `coverageOf`, which then only runs the files of + * a group that did not run together. By default a Vitest run of a whole group, when `coverageOf` is not given either. + */ + readonly coverageOfMany?: CoverageOfMany; /** Reasoned coverage exclusions from `web-tops.yaml`, passed to the default per-file run. */ readonly exclude?: readonly string[]; /** The random part of a merge-base copy's name; a test sets it to know the name. */ @@ -101,38 +114,88 @@ function refsCovered(map: CoverageMap, read: Reader): readonly FunctionRef[] { }); } -interface CoverRun { - readonly coverageOf: CoverageOf; - readonly read: Reader; - /** The coverage of a test file as it was at the merge base, or undefined when it has none or cannot run. */ - readonly baseCoverageOf: CoverageOf; +/** How many test files one run covers at most, so that a very large diff never overruns the length of a command line. */ +const GROUP = 100; + +function groupsOf(paths: readonly string[]): string[][] { + return Array.from({ length: Math.ceil(paths.length / GROUP) }, (_, at) => + paths.slice(at * GROUP, (at + 1) * GROUP), + ); } -const NONE: CoverageMap = new Map(); +interface Gathered { + readonly maps: readonly CoverageMap[]; + /** The files that could not run alone, in the order given. */ + readonly failed: readonly string[]; +} -async function headMap(test: ChangedFile, run: CoverRun): Promise { - return test.status === "deleted" ? NONE : run.coverageOf(test.path); +/** What the files cover, one run each: a file that cannot run is `failed`. */ +async function alone(group: readonly string[], one: CoverageOf): Promise { + const maps: CoverageMap[] = []; + const failed: string[] = []; + for (const path of group) { + const got = await one(path); + if (got === undefined) failed.push(path); + else maps.push(got); + } + return { maps, failed }; } -async function baseMap(test: ChangedFile, run: CoverRun): Promise { - return test.status === "added" ? NONE : ((await run.baseCoverageOf(test.path)) ?? NONE); +/** A group in one run when `many` is given, and file by file where that run gives nothing. */ +async function groupCoverage( + group: readonly string[], + one: CoverageOf, + many: CoverageOfMany | undefined, +): Promise { + const together = many === undefined ? undefined : await many(group); + return together === undefined ? alone(group, one) : { maps: [together], failed: [] }; } +/** + * What `paths` cover. Each group of them runs together when `many` is given; a group that does not run together (one + * of its files fails, or the run leaves nothing) runs file by file, as every file did before, so a file that cannot + * run is found and named whatever the others do. + */ +async function gather( + paths: readonly string[], + one: CoverageOf, + many: CoverageOfMany | undefined, +): Promise { + const found: Gathered[] = []; + for (const group of groupsOf(paths)) found.push(await groupCoverage(group, one, many)); + return { maps: found.flatMap((got) => got.maps), failed: found.flatMap((got) => got.failed) }; +} + +interface Sources { + readonly coverageOf: CoverageOf; + readonly coverageOfMany: CoverageOfMany | undefined; +} + +interface CoverRun extends Sources { + readonly read: Reader; + /** The merge-base versions of the changed test files, run beside the originals, as they covered then. */ + readonly baseMaps: (tests: readonly ChangedFile[]) => Promise; +} + +const byPathThenStartRef = (a: FunctionRef, b: FunctionRef): number => + a.path.localeCompare(b.path) || a.start - b.start; + /** * The functions every changed test file covers, as it is at HEAD and as it was at the merge base (so a test - * that dropped a call keeps the function it no longer calls in scope). A head version that cannot run is `fail`; - * a merge-base version that cannot run against the head is left out. + * that dropped a call keeps the function it no longer calls in scope). A head version that cannot run is `fail`, + * naming the first such file; a merge-base version that cannot run against the head is left out. The functions are + * those any of the runs covered, ordered by file and line. */ async function coveredFunctions(tests: readonly ChangedFile[], run: CoverRun): Promise { - const refs: FunctionRef[] = []; - for (const test of tests) { - const head = await headMap(test, run); - if (head === undefined) { - return { ok: false, reason: `could not run ${test.path} to find the functions it covers` }; - } - refs.push(...refsCovered(head, run.read), ...refsCovered(await baseMap(test, run), run.read)); + const heads = tests.filter((test) => test.status !== "deleted").map((test) => test.path); + const head = await gather(heads, run.coverageOf, run.coverageOfMany); + const [first] = head.failed; + if (first !== undefined) { + return { ok: false, reason: `could not run ${first} to find the functions it covers` }; } - return { ok: true, refs: uniqueByKey(refs) }; + const bases = await run.baseMaps(tests.filter((test) => test.status !== "added")); + const refs = [...head.maps, ...bases].flatMap((map) => refsCovered(map, run.read)); + return { ok: true, refs: [...uniqueByKey(refs)].sort(byPathThenStartRef) }; } const randomToken = (): string => randomBytes(6).toString("hex"); @@ -161,36 +224,74 @@ function createCopy(path: string, text: string): boolean { } } -/** - * The coverage of the merge-base copy of a test file, written beside the original so its imports resolve, run, - * and removed. Only a file this created is removed; a path that is already taken means no run (undefined). - */ -async function baseCoverage( +interface Copy { + /** The copy's path relative to the project, as the test runner is asked for it. */ + readonly name: string; + readonly path: string; +} + +/** The merge-base copy of one test file, written beside it so its imports resolve; undefined when the file is not at the merge base or the name is taken. */ +async function copyOf( options: ScopeOptions, exec: Exec, mergeBase: string, - coverageOf: CoverageOf, test: string, -): Promise { +): Promise { const shown = await showAt(exec, options.cwd, mergeBase, test); if (!shown.ok) return undefined; - const copy = copyName(test, (options.copyToken ?? randomToken)()); - const path = join(options.cwd, copy); - return createCopy(path, shown.value) ? runCopy(coverageOf, copy, path) : undefined; + const name = copyName(test, (options.copyToken ?? randomToken)()); + const path = join(options.cwd, name); + return createCopy(path, shown.value) ? { name, path } : undefined; } -/** Runs the coverage of a copy this created, and removes the copy whatever happens. */ -async function runCopy( - coverageOf: CoverageOf, - copy: string, - path: string, -): Promise { +/** A coverage source that returns undefined for a run that throws. */ +const quiet = + (run: (given: A) => Promise) => + async (given: A): Promise => { + try { + return await run(given); + } catch { + return undefined; + } + }; + +const quietMany = (many: CoverageOfMany | undefined): CoverageOfMany | undefined => + many === undefined ? undefined : quiet(many); + +/** Writes the merge-base copy of each test file that has one, adding each to `copies` as it is made. */ +async function copyAll( + copies: Copy[], + options: ScopeOptions, + exec: Exec, + mergeBase: string, + tests: readonly ChangedFile[], +): Promise { + for (const test of tests) { + const copy = await copyOf(options, exec, mergeBase, test.path); + if (copy !== undefined) copies.push(copy); + } +} + +/** + * The coverage of the merge-base versions of the changed test files, each written beside the original under a name + * that is never the original's, run together (file by file where that fails), and all removed whatever happens. Only a + * file this created is removed; a path that is already taken means no run for that file. + */ +async function baseCoverage( + options: ScopeOptions, + exec: Exec, + mergeBase: string, + sources: Sources, + tests: readonly ChangedFile[], +): Promise { + const copies: Copy[] = []; try { - return await coverageOf(copy); - } catch { - return undefined; + await copyAll(copies, options, exec, mergeBase, tests); + const names = copies.map((copy) => copy.name); + const got = await gather(names, quiet(sources.coverageOf), quietMany(sources.coverageOfMany)); + return got.maps; } finally { - rmSync(path, { force: true }); + for (const copy of copies) rmSync(copy.path, { force: true }); } } @@ -239,6 +340,21 @@ function scopeOf( }; } +/** A source for a group of files: the caller's, else Vitest on the group, unless the caller gave a source for one file only. */ +function manyOf(options: ScopeOptions, run: CoverageRun): CoverageOfMany | undefined { + if (options.coverageOfMany !== undefined) return options.coverageOfMany; + return options.coverageOf === undefined ? (group) => vitestCoverageOfMany(group, run) : undefined; +} + +/** Where coverage comes from: what the caller gave, else Vitest runs. */ +function sourcesOf(options: ScopeOptions, exec: Exec): Sources { + const run = { cwd: options.cwd, exec, exclude: options.exclude ?? [] }; + return { + coverageOf: options.coverageOf ?? ((test) => vitestCoverageOf(test, run)), + coverageOfMany: manyOf(options, run), + }; +} + async function assemble( options: ScopeOptions, exec: Exec, @@ -246,14 +362,13 @@ async function assemble( files: readonly ChangedFile[], ): Promise { const read = readerFor(options.cwd); - const coverageOf = - options.coverageOf ?? - ((test: string) => - vitestCoverageOf(test, { cwd: options.cwd, exec, exclude: options.exclude ?? [] })); - const baseCoverageOf = (test: string): Promise => - baseCoverage(options, exec, mergeBase, coverageOf, test); + const coverage = sourcesOf(options, exec); const tests = files.filter((file) => isTestFile(file.path)); - const covered = await coveredFunctions(tests, { coverageOf, read, baseCoverageOf }); + const covered = await coveredFunctions(tests, { + ...coverage, + read, + baseMaps: (changed) => baseCoverage(options, exec, mergeBase, coverage, changed), + }); if (!covered.ok) return { outcome: "fail", reason: covered.reason }; const scope = scopeOf(mergeBase, files, read, covered.refs); return scope.mutate.length === 0 From 64cf275d7fd2891b9ef254db5d6d5a75be21fd35 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 22:51:55 -0500 Subject: [PATCH 2/8] Say in section 6.4 how the changed test files are run for their coverage Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + WEB_TOPS.md | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97ab4b4..333a4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project are listed here. The protocol version and th ## [Unreleased] - Milestone M3 opens: `knip`, `secretlint` with its recommended preset and `eslint-plugin-react-hooks` are pinned exactly in §2, `package.json` and the lock, and the M3 paths are in every tool scope. §8.1 and §17 name the law fixtures `fixtures/laws/W-NN/{bad,clean}/`. (G0) +- The diff step finds what the changed test files cover with one Vitest run for their HEAD versions and one for their merge-base versions, not one run of each file alone, and runs file by file only where a group gives nothing. The functions in scope, the mutants and the verdicts are the same; the cost of the step falls with the number of changed test files. ## [0.3.0] - 2026-11-19 diff --git a/WEB_TOPS.md b/WEB_TOPS.md index 46ee713..98cf003 100644 --- a/WEB_TOPS.md +++ b/WEB_TOPS.md @@ -390,10 +390,11 @@ banned shortcuts, `!` and `as`, cheaper than handling the case. The profile key test file at the merge base or at HEAD**, so a change that only weakens a test, or drops the call to a function, is mutated. `webtops` computes the line ranges and passes them as Stryker `mutate` entries, `file:start-end` (confirmed on Stryker 10.1.1). It maps each changed test file to the - functions it covers by running that file alone under Vitest, with the gate's own configuration + functions it covers by running the changed test files under Vitest, with the gate's own configuration (the project's, coverage thresholds removed) and the gate's coverage include and exclude (§6.1), - once for the version at HEAD and once for the version at the merge base; a deleted test file runs - only the second. The functions that ran are added, with their whole ranges. Each range is then widened to the scored units it reaches into (§6.7), and a range that touches module-level code to all of the file's module-level lines, so the occurrence in a survivor's key does not depend on the diff. A HEAD version that + once for the versions at HEAD and once for the versions at the merge base, each in one run of up to 100 files (a function + is covered when any file ran it; a group whose run gives nothing runs file by file, so the file that cannot run is + found); a deleted test file is in the second only. The functions that ran are added, with their whole ranges. Each range is then widened to the scored units it reaches into (§6.7), and a range that touches module-level code to all of the file's module-level lines, so the occurrence in a survivor's key does not depend on the diff. A HEAD version that cannot run is `fail`; a merge-base version that cannot run against HEAD adds nothing. The diff is taken without rename detection, so a renamed file counts as new and every function in it is judged. A pure deletion belongs to the smallest function that holds the lines on both sides of From 1fca7c9c453263d05665eba02b508e5a0be6079c Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:25:03 -0500 Subject: [PATCH 3/8] Keep the quiet coverage source a plain function, so CRAP reads its statements Co-Authored-By: Claude Sonnet 5.5 --- src/diff/scope.ts | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/diff/scope.ts b/src/diff/scope.ts index 981a0c9..21680fc 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -244,19 +244,21 @@ async function copyOf( return createCopy(path, shown.value) ? { name, path } : undefined; } -/** A coverage source that returns undefined for a run that throws. */ -const quiet = - (run: (given: A) => Promise) => - async (given: A): Promise => { - try { - return await run(given); - } catch { - return undefined; - } - }; +/** What a coverage source gives, or undefined for a run that throws. */ +async function quietly( + run: (given: A) => Promise, + given: A, +): Promise { + try { + return await run(given); + } catch { + return undefined; + } +} -const quietMany = (many: CoverageOfMany | undefined): CoverageOfMany | undefined => - many === undefined ? undefined : quiet(many); +function quietMany(many: CoverageOfMany | undefined): CoverageOfMany | undefined { + return many === undefined ? undefined : (group) => quietly(many, group); +} /** Writes the merge-base copy of each test file that has one, adding each to `copies` as it is made. */ async function copyAll( @@ -288,7 +290,11 @@ async function baseCoverage( try { await copyAll(copies, options, exec, mergeBase, tests); const names = copies.map((copy) => copy.name); - const got = await gather(names, quiet(sources.coverageOf), quietMany(sources.coverageOfMany)); + const got = await gather( + names, + (file) => quietly(sources.coverageOf, file), + quietMany(sources.coverageOfMany), + ); return got.maps; } finally { for (const copy of copies) rmSync(copy.path, { force: true }); From 0aaf9bbe09e189655efbb45053b3179fb8f808ba Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:26:48 -0500 Subject: [PATCH 4/8] Pin the edges of the diff step that the batching puts in mutation scope A test that drives diffScope reaches the git, function and coverage readers, so their weak spots are scored with the rest. Each gets a case: what a path counts as a test or as source, a coverage entry with no location, a hit that a function only partly holds, functions with no body, TSX against TS, what git says when it fails, the arguments of the default Vitest runs and the removal of their directories, the name of a merge-base copy, and ranges that do not join across files. The module-level patterns of scope.ts move into the functions that use them. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/coverage.ts | 6 +- src/diff/functions.ts | 2 +- src/diff/pins.test.ts | 316 ++++++++++++++++++++++++++++++++++++++++++ src/diff/scope.ts | 10 +- 4 files changed, 324 insertions(+), 10 deletions(-) create mode 100644 src/diff/pins.test.ts diff --git a/src/diff/coverage.ts b/src/diff/coverage.ts index d894349..495436a 100644 --- a/src/diff/coverage.ts +++ b/src/diff/coverage.ts @@ -66,11 +66,9 @@ function hitsByFile( */ export function parseCoverage( value: unknown, - root?: string | readonly string[], + root: string | readonly string[] = [], ): CoverageMap | undefined { - return isRecord(value) - ? hitsByFile(value, typeof root === "string" ? [root] : (root ?? [])) - : undefined; + return isRecord(value) ? hitsByFile(value, typeof root === "string" ? [root] : root) : undefined; } /** The functions, of those the spans list, that ran: each hit goes to the smallest function-or-module span that holds it. */ diff --git a/src/diff/functions.ts b/src/diff/functions.ts index 8fa34e4..e092337 100644 --- a/src/diff/functions.ts +++ b/src/diff/functions.ts @@ -107,7 +107,7 @@ function kindOf(file: string): ts.ScriptKind { export function functionsIn(file: string, text: string): readonly FunctionSpan[] { const source = ts.createSourceFile(file, text, ts.ScriptTarget.ES2023, true, kindOf(file)); const module = `${file}::`; - const last = text.replace(/\n$/, "").split("\n").length; + const last = text.split("\n").length - (text.endsWith("\n") ? 1 : 0); const walk: Walk = { file, source, diff --git a/src/diff/pins.test.ts b/src/diff/pins.test.ts new file mode 100644 index 0000000..b3f319a --- /dev/null +++ b/src/diff/pins.test.ts @@ -0,0 +1,316 @@ +import { existsSync, rmSync } from "node:fs"; +import { basename, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { realExec } from "../gate/exec.ts"; +import type { Exec } from "../gate/types.ts"; +import { COVERAGE_INCLUDE, DEFAULT_COVERAGE_EXCLUDES } from "../layers/coverage/scope.ts"; +import { coveredSpans, parseCoverage } from "./coverage.ts"; +import { functionsIn, innermostAt } from "./functions.ts"; +import { mergeBaseOf } from "./git.ts"; +import { diffScope, isSourceFile, isTestFile } from "./scope.ts"; +import { cleanRepos, commit, git, makeRepo, write } from "./test-support.ts"; +import type { CoverageMap } from "./types.ts"; + +// What the diff step does at its edges, pinned one fact at a time. + +afterEach(cleanRepos); + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "", +].join("\n"); +const TEST = 'import { f, g } from "./m.ts";\nf(1);\ng(2);\n'; + +const ran = (f: boolean, g: boolean): CoverageMap => + new Map([ + [ + "src/m.ts", + [ + { start: 1, end: 3, hits: f ? 1 : 0 }, + { start: 4, end: 6, hits: g ? 1 : 0 }, + ], + ], + ]); + +describe("which paths are tests and which are source", () => { + it.each([ + "a.test.ts", + "a.spec.ts", + "a.test.tsx", + "a.test.mts", + "a.spec.cts", + "src/__tests__/a.ts", + "__tests__/a.ts", + ])("%s is a test", (path) => { + expect(isTestFile(path)).toBe(true); + expect(isSourceFile(path)).toBe(false); + }); + + it.each(["a.ts", "a.tsx", "a.mts", "a.cts", "a.test.tsy.ts", "a.d.ts.ts", "src/__tests__x/a.ts"])( + "%s is source", + (path) => { + expect(isSourceFile(path)).toBe(true); + expect(isTestFile(path)).toBe(false); + }, + ); + + it.each(["a.js", "a.ts.bak", "a.test.js", "src/__tests__/a.js", "a.d.ts", "a.d.mts", "a.d.cts"])( + "%s is neither", + (path) => { + expect(isSourceFile(path)).toBe(false); + expect(isTestFile(path)).toBe(false); + }, + ); +}); + +describe("coverage maps", () => { + it("refuses a file whose functions have no location, and keeps one that has them", () => { + const entry = { loc: { start: { line: 1 }, end: { line: 2 } } }; + expect(parseCoverage({ "a.ts": { fnMap: { 0: {} }, f: { 0: 1 } } })).toBeUndefined(); + expect(parseCoverage({ "a.ts": { fnMap: { 0: entry }, f: { 0: 1 } } })?.get("a.ts")).toEqual([ + { start: 1, end: 2, hits: 1 }, + ]); + }); + + it("takes no work-tree prefix off a path when it is given none", () => { + const file = { fnMap: { 0: { loc: { start: { line: 1 }, end: { line: 2 } } } }, f: { 0: 1 } }; + expect([...(parseCoverage({ "/work/a.ts": file })?.keys() ?? [])]).toEqual(["/work/a.ts"]); + }); + + const spans = [ + { key: "a.ts::", start: 1, end: 20 }, + { key: "a.ts::f", start: 3, end: 10 }, + { key: "a.ts::g", start: 12, end: 15 }, + ]; + + it("never lists the module as a covered function, whatever ran in it", () => { + expect(coveredSpans(spans, [{ start: 1, end: 20, hits: 3 }])).toEqual([]); + }); + + it("gives a hit to a function only when the function holds all of it", () => { + expect(coveredSpans(spans, [{ start: 5, end: 11, hits: 1 }])).toEqual([]); + expect(coveredSpans(spans, [{ start: 5, end: 8, hits: 1 }]).map((span) => span.key)).toEqual([ + "a.ts::f", + ]); + }); +}); + +describe("functions of a file", () => { + it("leaves out what has no body: overloads, abstract members and signatures", () => { + const text = [ + "export function ov(a: string): void;", + "export function ov(a: number): void;", + "export function ov(a: string | number): void {}", + "abstract class A {", + " abstract m(): void;", + "}", + "interface I {", + " n(): void;", + "}", + "namespace N {", + " export const x = 1;", + "}", + "", + ].join("\n"); + expect(functionsIn("a.ts", text).map((span) => span.key)).toEqual([ + "a.ts::", + "a.ts::ov", + ]); + }); + + it("reads a .ts file as TypeScript, where a generic arrow is not JSX", () => { + const keys = functionsIn("a.ts", "const f = (a: T) => a;\nconst h = (a: T) => a;\n"); + expect(keys.map((span) => span.key)).toEqual(["a.ts::", "a.ts::f", "a.ts::h"]); + }); + + it("reads a .tsx file as TSX", () => { + const keys = functionsIn("a.tsx", "const v =
{[1].map((x) => x)}
;\n"); + expect(keys.map((span) => span.key)).toEqual(["a.tsx::", "a.tsx::#1"]); + }); + + it("ends the module where the text ends, with or without a last newline", () => { + expect(functionsIn("a.ts", "const a = 1;\nconst b = 2;\n")[0]?.end).toBe(2); + expect(functionsIn("a.ts", "const a = 1;\nconst b = 2;")[0]?.end).toBe(2); + expect(functionsIn("a.ts", "const a = 1;\n\n")[0]?.end).toBe(2); + }); + + it("picks the innermost span whatever order they come in, and the later of two of one size", () => { + const outer = { key: "o", start: 1, end: 10 }; + const inner = { key: "i", start: 3, end: 4 }; + expect(innermostAt([outer, inner], 3)?.key).toBe("i"); + expect(innermostAt([inner, outer], 3)?.key).toBe("i"); + expect(innermostAt([{ ...inner, key: "x" }, inner], 3)?.key).toBe("i"); + }); +}); + +describe("what git says when it fails", () => { + const failing = + (stderr: string, stdout = "", code = 1): Exec => + () => + Promise.resolve({ code, stdout, stderr }); + + it("is its own words, trimmed", async () => { + expect(await mergeBaseOf(failing(" boom \n"), "/x", "main")).toEqual({ + ok: false, + reason: "not a git work tree: boom", + }); + }); + + it("is the command that failed when git said nothing", async () => { + expect(await mergeBaseOf(failing(""), "/x", "main")).toEqual({ + ok: false, + reason: "not a git work tree: git rev-parse --is-inside-work-tree failed", + }); + }); + + it("is that there is no merge base when git printed none", async () => { + expect(await mergeBaseOf(failing("", "", 0), "/x", "main")).toEqual({ + ok: false, + reason: "there is no merge base between HEAD and main", + }); + }); +}); + +/** What the fake Vitest does with a run: fail a run of two or more files, else write a report that covers f. */ +function fakeVitest(args: readonly string[]): { code: number; stdout: string; stderr: string } { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + const files = args.filter((arg) => arg.endsWith(".test.ts")); + if (files.length > 1) return { code: 1, stdout: "", stderr: "" }; + const loc = { start: { line: 1 }, end: { line: 3 } }; + const report = { "src/m.ts": { fnMap: { 0: { loc } }, f: { 0: 1 } } }; + write(flag.slice("--coverage.reportsDirectory=".length), { + "coverage-final.json": JSON.stringify(report), + }); + return { code: 0, stdout: "", stderr: "" }; +} + +/** Git runs for real; Vitest is a fake that keeps its arguments. */ +function spied(): { exec: Exec; vitest: string[][]; shows: string[][] } { + const vitest: string[][] = []; + const shows: string[][] = []; + const exec: Exec = (command, args, options) => { + if (command !== "git") { + vitest.push([...args]); + return Promise.resolve(fakeVitest(args)); + } + if (args[0] === "show") shows.push([...args]); + return realExec(command, args, options); + }; + return { exec, vitest, shows }; +} + +describe("the default runs of Vitest", () => { + function added(): string { + const dir = makeRepo({ "src/m.ts": M }); + write(dir, { "src/a.test.ts": TEST, "src/b.test.ts": TEST }); + git(dir, "add", "-A"); + return dir; + } + + it("runs both files together first, then each alone where that fails, under the gate's own options", async () => { + const { exec, vitest } = spied(); + const result = await diffScope({ cwd: added(), base: "main", exec }); + expect(vitest.map((args) => args.filter((arg) => arg.endsWith(".test.ts")))).toEqual([ + ["src/a.test.ts", "src/b.test.ts"], + ["src/a.test.ts"], + ["src/b.test.ts"], + ]); + const first = vitest[0] ?? []; + expect(first.slice(0, 3)).toEqual(["run", "src/a.test.ts", "src/b.test.ts"]); + expect(first.filter((arg) => arg.startsWith("--coverage.exclude="))).toEqual( + DEFAULT_COVERAGE_EXCLUDES.map((glob) => `--coverage.exclude=${glob}`), + ); + expect(first).toContain(`--coverage.include=${COVERAGE_INCLUDE}`); + expect(first).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length); + expect(result.outcome).toBe("scope"); + }); + + it("adds the exclusions the project gives, and removes its temporary directories", async () => { + const { exec, vitest } = spied(); + await diffScope({ cwd: added(), base: "main", exec, exclude: ["gen/**"] }); + const first = vitest[0] ?? []; + expect(first).toContain("--coverage.exclude=gen/**"); + expect(first).toHaveLength(10 + DEFAULT_COVERAGE_EXCLUDES.length); + const flag = first.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + const out = flag.slice("--coverage.reportsDirectory=".length); + expect(basename(out).startsWith("wt-cov-")).toBe(true); + expect(existsSync(out)).toBe(false); + }); + + it("asks git for the merge-base version of a modified test, and of no added one", async () => { + const { exec, shows } = spied(); + await diffScope({ cwd: added(), base: "main", exec }); + expect(shows).toEqual([]); + const dir = makeRepo({ "src/m.ts": M, "src/a.test.ts": TEST }); + git(dir, "switch", "-q", "-c", "work"); + write(dir, { "src/a.test.ts": `${TEST}f(3);\n`, "src/b.test.ts": TEST }); + commit(dir, "work"); + const second = spied(); + await diffScope({ cwd: dir, base: "main", exec: second.exec }); + expect(second.shows).toHaveLength(1); + expect(second.shows[0]?.[1]).toMatch(/:src\/a\.test\.ts$/); + }); +}); + +describe("the merge-base copies", () => { + it("are named before the extension for a test with no marker, after its last dot", async () => { + const dir = makeRepo({ "src/m.ts": M, "src/__tests__/a.b.ts": TEST }); + git(dir, "switch", "-q", "-c", "work"); + write(dir, { "src/__tests__/a.b.ts": `${TEST}f(3);\n` }); + commit(dir, "work"); + const asked: string[] = []; + await diffScope({ + copyToken: () => "tok", + cwd: dir, + base: "main", + coverageOf: (file) => { + asked.push(file); + return Promise.resolve(ran(true, false)); + }, + }); + expect(asked).toEqual(["src/__tests__/a.b.ts", "src/__tests__/a.b.wt-base-tok.ts"]); + }); + + it("are removed even when the run took the copy away itself", async () => { + const dir = makeRepo({ "src/m.ts": M, "src/a.test.ts": TEST }); + git(dir, "switch", "-q", "-c", "work"); + write(dir, { "src/a.test.ts": `${TEST}f(3);\n` }); + commit(dir, "work"); + const result = await diffScope({ + copyToken: () => "tok", + cwd: dir, + base: "main", + coverageOf: (file) => { + if (file.includes(".wt-base-")) rmSync(join(dir, file)); + return Promise.resolve(ran(true, false)); + }, + }); + expect(result.outcome).toBe("scope"); + }); +}); + +describe("the ranges to mutate", () => { + it("are not joined across two files, however close the last line of one and the first of the next", async () => { + const lines = (n: number, mark: string): string => + `${Array.from({ length: n }, (_, at) => `export const v${String(at)}${mark} = ${String(at)};`).join("\n")}\n`; + const dir = makeRepo({ "src/a.ts": lines(5, "a"), "src/b.ts": lines(8, "b") }); + git(dir, "switch", "-q", "-c", "work"); + const edit = (text: string, from: number, to: number): string => + text + .split("\n") + .map((line, at) => (at + 1 >= from && at + 1 <= to ? `${line} // edited` : line)) + .join("\n"); + write(dir, { "src/a.ts": edit(lines(5, "a"), 3, 5), "src/b.ts": edit(lines(8, "b"), 6, 8) }); + commit(dir, "work"); + const result = await diffScope({ cwd: dir, base: "main" }); + expect(result.outcome === "scope" ? result.scope.mutate : []).toEqual([ + "src/a.ts:3-5", + "src/b.ts:6-8", + ]); + }); +}); diff --git a/src/diff/scope.ts b/src/diff/scope.ts index 21680fc..9b7a341 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -56,13 +56,13 @@ type Covered = type Reader = (path: string) => string | undefined; -const TEST = /\.(test|spec)\.[cm]?tsx?$|(^|\/)__tests__\//; -const SOURCE = /\.[cm]?tsx?$/; -const DECLARATION = /\.d\.[cm]?ts$/; +const hasScriptExtension = (path: string): boolean => /\.[cm]?tsx?$/.test(path); +const isDeclaration = (path: string): boolean => /\.d\.[cm]?ts$/.test(path); -export const isTestFile = (path: string): boolean => TEST.test(path) && SOURCE.test(path); +export const isTestFile = (path: string): boolean => + /\.(test|spec)\.[cm]?tsx?$|(^|\/)__tests__\//.test(path) && hasScriptExtension(path); export const isSourceFile = (path: string): boolean => - SOURCE.test(path) && !DECLARATION.test(path) && !TEST.test(path); + hasScriptExtension(path) && !isDeclaration(path) && !isTestFile(path); function readerFor(cwd: string): Reader { return (path) => { From e71de9422612232c93fb5769678cdf26b58b7bfc Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:52:56 -0500 Subject: [PATCH 5/8] Pin the default Vitest options and a parameter default, and ledger four equivalent mutants Co-Authored-By: Claude Sonnet 5.5 --- .web-tops/survivors.yaml | 39 ++++++++++++++++++++++++++++++++++ src/diff/coverage-many.test.ts | 24 +++++++++++++++++++++ src/diff/pins.test.ts | 14 +++++++++++- 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/.web-tops/survivors.yaml b/.web-tops/survivors.yaml index c73d8dc..e77e35f 100644 --- a/.web-tops/survivors.yaml +++ b/.web-tops/survivors.yaml @@ -135,6 +135,45 @@ survivors: reason: document.toJS throws only Error instances (the YAML library), so the fallback text is a dead branch. owner: web-platform + - file: src/diff/coverage.ts + function: src/diff/coverage.ts::innermostHolding + mutator: ConditionalExpression + original: "span.start <= hit.start" + replacement: "true" + occurrence: 1 + reason: innermostAt then keeps only spans that hold the line of the hit's start, + which are exactly the spans that start at or before it, so the first test + of the filter changes nothing. + owner: web-platform + - file: src/diff/functions.ts + function: src/diff/functions.ts::boundName + mutator: ConditionalExpression + original: "parent.initializer === node" + replacement: "true" + occurrence: 1 + reason: "A function node is the child of a variable, property assignment or + property declaration only as its initializer: its other children are + names, types and decorators, none of them function nodes." + owner: web-platform + - file: src/diff/functions.ts + function: src/diff/functions.ts::declaredName + mutator: ConditionalExpression + original: "ts.isArrowFunction(node)" + replacement: "false" + occurrence: 1 + reason: An arrow function has no name, so the next line gives undefined for it + as well; the test exists for the type of the node, not for the result. + owner: web-platform + - file: src/diff/scope.ts + function: src/diff/scope.ts::scopeOf + mutator: ConditionalExpression + original: "file.status !== \"deleted\"" + replacement: "true" + occurrence: 1 + reason: A deleted file has no added lines, so it adds no range to mutate and no + function to the changed ones, whether or not it is counted among the + sources. + owner: web-platform - file: src/gate/adapter.ts function: src/gate/adapter.ts::classCap mutator: StringLiteral diff --git a/src/diff/coverage-many.test.ts b/src/diff/coverage-many.test.ts index 8974952..8267438 100644 --- a/src/diff/coverage-many.test.ts +++ b/src/diff/coverage-many.test.ts @@ -2,6 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import type { Exec } from "../gate/types.ts"; +import { DEFAULT_COVERAGE_EXCLUDES } from "../layers/coverage/scope.ts"; import { vitestCoverageOf, vitestCoverageOfMany } from "./coverage.ts"; const REPORT = { @@ -36,6 +37,29 @@ describe("vitestCoverageOfMany", () => { expect(map?.get("src/a.ts")).toEqual([{ start: 1, end: 3, hits: 1 }]); }); + it("gives the gate's own options and nothing else, then the extra arguments it is given", async () => { + const log: string[][] = []; + await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped(log), + }); + const plain = (log[0] ?? []).slice(1); + expect(plain).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length); + expect(plain.filter((arg) => arg.startsWith("--coverage.exclude="))).toEqual( + DEFAULT_COVERAGE_EXCLUDES.map((glob) => `--coverage.exclude=${glob}`), + ); + await vitestCoverageOfMany(["src/a.test.ts"], { + cwd: "/repo", + exec: stamped(log), + exclude: ["gen/**"], + args: ["--reporter=dot"], + }); + const extra = (log[1] ?? []).slice(1); + expect(extra.at(-1)).toBe("--reporter=dot"); + expect(extra).toContain("--coverage.exclude=gen/**"); + expect(extra).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length - 1 + 2); + }); + it("is undefined when the run fails, whichever file is the cause", async () => { const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { cwd: "/repo", diff --git a/src/diff/pins.test.ts b/src/diff/pins.test.ts index b3f319a..a262568 100644 --- a/src/diff/pins.test.ts +++ b/src/diff/pins.test.ts @@ -79,7 +79,9 @@ describe("coverage maps", () => { it("takes no work-tree prefix off a path when it is given none", () => { const file = { fnMap: { 0: { loc: { start: { line: 1 }, end: { line: 2 } } } }, f: { 0: 1 } }; - expect([...(parseCoverage({ "/work/a.ts": file })?.keys() ?? [])]).toEqual(["/work/a.ts"]); + const paths = (path: string): string[] => [...(parseCoverage({ [path]: file })?.keys() ?? [])]; + expect(paths("/work/a.ts")).toEqual(["/work/a.ts"]); + expect(paths("Stryker was here/a.ts")).toEqual(["Stryker was here/a.ts"]); }); const spans = [ @@ -123,6 +125,16 @@ describe("functions of a file", () => { ]); }); + it("names a function only by a variable, property assignment or property declaration it initializes", () => { + const text = "export function f(a = () => 1) {}\nconst { x = () => 2 } = o;\n"; + expect(functionsIn("a.ts", text).map((span) => span.key)).toEqual([ + "a.ts::", + "a.ts::f", + "a.ts::f#1", + "a.ts::#1", + ]); + }); + it("reads a .ts file as TypeScript, where a generic arrow is not JSX", () => { const keys = functionsIn("a.ts", "const f = (a: T) => a;\nconst h = (a: T) => a;\n"); expect(keys.map((span) => span.key)).toEqual(["a.ts::", "a.ts::f", "a.ts::h"]); From 40599b0f9efe37ac4dc3367cbb045617dbe1bb9e Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 03:13:19 -0500 Subject: [PATCH 6/8] Keep the tests of the diff step cheap for the mutants they cover Every mutant of the diff step re-runs each test that covers it. The batching tests ran real git for each case, and the real-Vitest pin ran three Vitest runs in-process, so a mutant cost tens of seconds on a three-CPU runner and the macOS leg stopped at the 30-minute mutation budget. Git is now a fake that answers from a diff text, in every test but the older ones, and the real-Vitest pin takes both scopes in a child process, where nothing it runs counts as coverage of the diff step. The cases are the same. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/batch.test.ts | 68 ++++++++++++------------ src/diff/pins.test.ts | 103 +++++++++++++++++------------------- src/diff/real-batch.test.ts | 92 ++++++++++++++++++++------------ src/diff/test-support.ts | 78 +++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 120 deletions(-) diff --git a/src/diff/batch.test.ts b/src/diff/batch.test.ts index fe5ab93..967262f 100644 --- a/src/diff/batch.test.ts +++ b/src/diff/batch.test.ts @@ -1,11 +1,10 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { realExec } from "../gate/exec.ts"; import type { Exec } from "../gate/types.ts"; import type { CoverageOf, CoverageOfMany } from "./scope.ts"; import { diffScope } from "./scope.ts"; -import { cleanRepos, commit, git, makeRepo, write } from "./test-support.ts"; +import { cleanRepos, diffOf, fakeGit, plainDir } from "./test-support.ts"; import type { CoverageMap, ScopeResult } from "./types.ts"; // The changed test files are run together, once, and file by file only where that gives nothing: the functions in @@ -37,19 +36,21 @@ const ran = (f: boolean, g: boolean): CoverageMap => ], ]); -function added(...names: string[]): string { - const dir = makeRepo({ "src/m.ts": M }); - write(dir, Object.fromEntries(names.map((name) => [name, WEAK]))); - git(dir, "add", "-A"); - return dir; +/** A project with src/m.ts and the given test files added, and the git that would say so. */ +function added(...names: string[]): { cwd: string; exec: Exec } { + const cwd = plainDir({ "src/m.ts": M, ...Object.fromEntries(names.map((name) => [name, WEAK])) }); + const exec = fakeGit(diffOf(names.map((path) => ({ path, kind: "added" })))); + return { cwd, exec }; } -function weakened(): string { - const dir = makeRepo({ "src/m.ts": M, "src/m.test.ts": STRONG }); - git(dir, "switch", "-q", "-c", "weaken"); - write(dir, { "src/m.test.ts": WEAK, "src/n.test.ts": WEAK }); - commit(dir, "weaken one test and add another"); - return dir; +/** src/m.test.ts weakened and src/n.test.ts added, with the strong version of the first at the merge base. */ +function weakened(): { cwd: string; exec: Exec } { + const cwd = plainDir({ "src/m.ts": M, "src/m.test.ts": WEAK, "src/n.test.ts": WEAK }); + const diff = diffOf([ + { path: "src/m.test.ts", kind: "modified" }, + { path: "src/n.test.ts", kind: "added" }, + ]); + return { cwd, exec: fakeGit(diff, { "src/m.test.ts": STRONG }) }; } const keys = (result: ScopeResult): string[] => @@ -60,7 +61,7 @@ describe("changed test files run together", () => { const together: (readonly string[])[] = []; const alone: string[] = []; const result = await diffScope({ - cwd: added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + ...added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), base: "main", coverageOfMany: (group) => { together.push(group); @@ -79,7 +80,7 @@ describe("changed test files run together", () => { it("runs them one by one, in order, when the run together gives nothing, and adds up what each covered", async () => { const alone: string[] = []; const result = await diffScope({ - cwd: added("src/a.test.ts", "src/b.test.ts"), + ...added("src/a.test.ts", "src/b.test.ts"), base: "main", coverageOfMany: () => Promise.resolve(undefined), coverageOf: (file) => { @@ -93,7 +94,7 @@ describe("changed test files run together", () => { it("fails naming the first file that cannot run alone, the later ones unnamed", async () => { const result = await diffScope({ - cwd: added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + ...added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), base: "main", coverageOfMany: () => Promise.resolve(undefined), coverageOf: (file) => Promise.resolve(file === "src/a.test.ts" ? ran(true, true) : undefined), @@ -105,16 +106,16 @@ describe("changed test files run together", () => { }); it("is the same scope as running each file alone", async () => { - const dir = added("src/a.test.ts", "src/b.test.ts"); + const project = added("src/a.test.ts", "src/b.test.ts"); const maps: Record = { "src/a.test.ts": ran(true, false), "src/b.test.ts": ran(false, true), }; const alone: CoverageOf = (file) => Promise.resolve(maps[file]); const many: CoverageOfMany = () => Promise.resolve(ran(true, true)); - const one = await diffScope({ cwd: dir, base: "main", coverageOf: alone }); + const one = await diffScope({ ...project, base: "main", coverageOf: alone }); const all = await diffScope({ - cwd: dir, + ...project, base: "main", coverageOfMany: many, coverageOf: alone, @@ -123,17 +124,17 @@ describe("changed test files run together", () => { }); it("runs the merge-base versions together too, as copies that are gone afterwards", async () => { - const dir = weakened(); + const project = weakened(); const together: (readonly string[])[] = []; const seen: string[] = []; const result = await diffScope({ + ...project, copyToken: () => "tok", - cwd: dir, base: "main", coverageOf: () => Promise.resolve(undefined), coverageOfMany: (group) => { together.push(group); - for (const file of group) seen.push(existsSync(join(dir, file)) ? file : ""); + for (const file of group) seen.push(existsSync(join(project.cwd, file)) ? file : ""); return Promise.resolve( group.some((file) => file.includes(".wt-base-")) ? ran(true, true) : ran(true, false), ); @@ -142,16 +143,16 @@ describe("changed test files run together", () => { expect(together).toEqual([["src/m.test.ts", "src/n.test.ts"], ["src/m.wt-base-tok.test.ts"]]); expect(seen).toEqual(["src/m.test.ts", "src/n.test.ts", "src/m.wt-base-tok.test.ts"]); expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); - expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); }); it("holds the merge-base version in the copy, and runs a copy that fails together alone, leaving it out", async () => { - const dir = weakened(); + const project = weakened(); const alone: string[] = []; const text: Record = {}; const result = await diffScope({ + ...project, copyToken: () => "tok", - cwd: dir, base: "main", coverageOfMany: (group) => Promise.resolve( @@ -159,21 +160,21 @@ describe("changed test files run together", () => { ), coverageOf: (file) => { alone.push(file); - text[file] = readFileSync(join(dir, file), "utf8"); + text[file] = readFileSync(join(project.cwd, file), "utf8"); return Promise.resolve(undefined); }, }); expect(alone).toEqual(["src/m.wt-base-tok.test.ts"]); expect(text["src/m.wt-base-tok.test.ts"]).toBe(STRONG.trim()); expect(keys(result)).toEqual(["src/m.ts::f"]); - expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); }); it("leaves a merge-base version out when its runs throw, and still removes the copy", async () => { - const dir = weakened(); + const project = weakened(); const result = await diffScope({ + ...project, copyToken: () => "tok", - cwd: dir, base: "main", coverageOfMany: (group) => { if (group.some((file) => file.includes(".wt-base-"))) throw new Error("the run died"); @@ -184,14 +185,14 @@ describe("changed test files run together", () => { }, }); expect(keys(result)).toEqual(["src/m.ts::f"]); - expect(existsSync(join(dir, "src/m.wt-base-tok.test.ts"))).toBe(false); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); }); it("runs at most 100 files at once", async () => { const names = Array.from({ length: 205 }, (_, at) => `src/t${String(at)}.test.ts`); const sizes: number[] = []; await diffScope({ - cwd: added(...names), + ...added(...names), base: "main", coverageOfMany: (group) => { sizes.push(group.length); @@ -203,12 +204,13 @@ describe("changed test files run together", () => { it("runs no group when it is given a source for one file only", async () => { const asked: string[] = []; + const project = added("src/a.test.ts", "src/b.test.ts"); const exec: Exec = (command, args, options) => { asked.push(command); - return realExec(command, args, options); + return project.exec(command, args, options); }; const result = await diffScope({ - cwd: added("src/a.test.ts", "src/b.test.ts"), + cwd: project.cwd, base: "main", exec, coverageOf: () => Promise.resolve(ran(true, false)), diff --git a/src/diff/pins.test.ts b/src/diff/pins.test.ts index a262568..3671929 100644 --- a/src/diff/pins.test.ts +++ b/src/diff/pins.test.ts @@ -1,14 +1,13 @@ import { existsSync, rmSync } from "node:fs"; import { basename, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { realExec } from "../gate/exec.ts"; import type { Exec } from "../gate/types.ts"; import { COVERAGE_INCLUDE, DEFAULT_COVERAGE_EXCLUDES } from "../layers/coverage/scope.ts"; import { coveredSpans, parseCoverage } from "./coverage.ts"; import { functionsIn, innermostAt } from "./functions.ts"; import { mergeBaseOf } from "./git.ts"; import { diffScope, isSourceFile, isTestFile } from "./scope.ts"; -import { cleanRepos, commit, git, makeRepo, write } from "./test-support.ts"; +import { cleanRepos, diffOf, FAKE_BASE, fakeGit, plainDir, write } from "./test-support.ts"; import type { CoverageMap } from "./types.ts"; // What the diff step does at its edges, pinned one fact at a time. @@ -201,32 +200,32 @@ function fakeVitest(args: readonly string[]): { code: number; stdout: string; st return { code: 0, stdout: "", stderr: "" }; } -/** Git runs for real; Vitest is a fake that keeps its arguments. */ -function spied(): { exec: Exec; vitest: string[][]; shows: string[][] } { +/** Git is the given fake; Vitest is a fake that keeps its arguments. */ +function spied(git: Exec): { exec: Exec; vitest: string[][] } { const vitest: string[][] = []; - const shows: string[][] = []; const exec: Exec = (command, args, options) => { - if (command !== "git") { - vitest.push([...args]); - return Promise.resolve(fakeVitest(args)); - } - if (args[0] === "show") shows.push([...args]); - return realExec(command, args, options); + if (command === "git") return git(command, args, options); + vitest.push([...args]); + return Promise.resolve(fakeVitest(args)); }; - return { exec, vitest, shows }; + return { exec, vitest }; } describe("the default runs of Vitest", () => { - function added(): string { - const dir = makeRepo({ "src/m.ts": M }); - write(dir, { "src/a.test.ts": TEST, "src/b.test.ts": TEST }); - git(dir, "add", "-A"); - return dir; - } + const added = (): { cwd: string; git: Exec } => ({ + cwd: plainDir({ "src/m.ts": M, "src/a.test.ts": TEST, "src/b.test.ts": TEST }), + git: fakeGit( + diffOf([ + { path: "src/a.test.ts", kind: "added" }, + { path: "src/b.test.ts", kind: "added" }, + ]), + ), + }); it("runs both files together first, then each alone where that fails, under the gate's own options", async () => { - const { exec, vitest } = spied(); - const result = await diffScope({ cwd: added(), base: "main", exec }); + const project = added(); + const { exec, vitest } = spied(project.git); + const result = await diffScope({ cwd: project.cwd, base: "main", exec }); expect(vitest.map((args) => args.filter((arg) => arg.endsWith(".test.ts")))).toEqual([ ["src/a.test.ts", "src/b.test.ts"], ["src/a.test.ts"], @@ -243,8 +242,9 @@ describe("the default runs of Vitest", () => { }); it("adds the exclusions the project gives, and removes its temporary directories", async () => { - const { exec, vitest } = spied(); - await diffScope({ cwd: added(), base: "main", exec, exclude: ["gen/**"] }); + const project = added(); + const { exec, vitest } = spied(project.git); + await diffScope({ cwd: project.cwd, base: "main", exec, exclude: ["gen/**"] }); const first = vitest[0] ?? []; expect(first).toContain("--coverage.exclude=gen/**"); expect(first).toHaveLength(10 + DEFAULT_COVERAGE_EXCLUDES.length); @@ -255,30 +255,30 @@ describe("the default runs of Vitest", () => { }); it("asks git for the merge-base version of a modified test, and of no added one", async () => { - const { exec, shows } = spied(); - await diffScope({ cwd: added(), base: "main", exec }); - expect(shows).toEqual([]); - const dir = makeRepo({ "src/m.ts": M, "src/a.test.ts": TEST }); - git(dir, "switch", "-q", "-c", "work"); - write(dir, { "src/a.test.ts": `${TEST}f(3);\n`, "src/b.test.ts": TEST }); - commit(dir, "work"); - const second = spied(); - await diffScope({ cwd: dir, base: "main", exec: second.exec }); - expect(second.shows).toHaveLength(1); - expect(second.shows[0]?.[1]).toMatch(/:src\/a\.test\.ts$/); + const seen: string[][] = []; + const cwd = plainDir({ "src/m.ts": M, "src/a.test.ts": TEST, "src/b.test.ts": TEST }); + const diff = diffOf([ + { path: "src/a.test.ts", kind: "modified" }, + { path: "src/b.test.ts", kind: "added" }, + ]); + const { exec } = spied(fakeGit(diff, { "src/a.test.ts": TEST }, seen)); + await diffScope({ cwd, base: "main", exec }); + const shows = seen.filter((args) => args[0] === "show"); + expect(shows).toEqual([["show", `${FAKE_BASE}:src/a.test.ts`]]); }); }); describe("the merge-base copies", () => { + const modified = (path: string): { cwd: string; exec: Exec } => ({ + cwd: plainDir({ "src/m.ts": M, [path]: TEST }), + exec: fakeGit(diffOf([{ path, kind: "modified" }]), { [path]: TEST }), + }); + it("are named before the extension for a test with no marker, after its last dot", async () => { - const dir = makeRepo({ "src/m.ts": M, "src/__tests__/a.b.ts": TEST }); - git(dir, "switch", "-q", "-c", "work"); - write(dir, { "src/__tests__/a.b.ts": `${TEST}f(3);\n` }); - commit(dir, "work"); const asked: string[] = []; await diffScope({ + ...modified("src/__tests__/a.b.ts"), copyToken: () => "tok", - cwd: dir, base: "main", coverageOf: (file) => { asked.push(file); @@ -289,16 +289,13 @@ describe("the merge-base copies", () => { }); it("are removed even when the run took the copy away itself", async () => { - const dir = makeRepo({ "src/m.ts": M, "src/a.test.ts": TEST }); - git(dir, "switch", "-q", "-c", "work"); - write(dir, { "src/a.test.ts": `${TEST}f(3);\n` }); - commit(dir, "work"); + const project = modified("src/a.test.ts"); const result = await diffScope({ + ...project, copyToken: () => "tok", - cwd: dir, base: "main", coverageOf: (file) => { - if (file.includes(".wt-base-")) rmSync(join(dir, file)); + if (file.includes(".wt-base-")) rmSync(join(project.cwd, file)); return Promise.resolve(ran(true, false)); }, }); @@ -310,16 +307,14 @@ describe("the ranges to mutate", () => { it("are not joined across two files, however close the last line of one and the first of the next", async () => { const lines = (n: number, mark: string): string => `${Array.from({ length: n }, (_, at) => `export const v${String(at)}${mark} = ${String(at)};`).join("\n")}\n`; - const dir = makeRepo({ "src/a.ts": lines(5, "a"), "src/b.ts": lines(8, "b") }); - git(dir, "switch", "-q", "-c", "work"); - const edit = (text: string, from: number, to: number): string => - text - .split("\n") - .map((line, at) => (at + 1 >= from && at + 1 <= to ? `${line} // edited` : line)) - .join("\n"); - write(dir, { "src/a.ts": edit(lines(5, "a"), 3, 5), "src/b.ts": edit(lines(8, "b"), 6, 8) }); - commit(dir, "work"); - const result = await diffScope({ cwd: dir, base: "main" }); + const cwd = plainDir({ "src/a.ts": lines(5, "a"), "src/b.ts": lines(8, "b") }); + const exec = fakeGit( + diffOf([ + { path: "src/a.ts", kind: "modified", range: [3, 5] }, + { path: "src/b.ts", kind: "modified", range: [6, 8] }, + ]), + ); + const result = await diffScope({ cwd, base: "main", exec }); expect(result.outcome === "scope" ? result.scope.mutate : []).toEqual([ "src/a.ts:3-5", "src/b.ts:6-8", diff --git a/src/diff/real-batch.test.ts b/src/diff/real-batch.test.ts index b5e34ce..b1ea258 100644 --- a/src/diff/real-batch.test.ts +++ b/src/diff/real-batch.test.ts @@ -1,14 +1,15 @@ -import { rmSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { realExec } from "../gate/exec.ts"; import { REAL_TOOL_TIMEOUT } from "../gate/test-support-timeouts.ts"; import { realProject } from "../layers/tests/testkit.ts"; -import { vitestCoverageOf } from "./coverage.ts"; -import { diffScope } from "./scope.ts"; import { commit, git, write } from "./test-support.ts"; // With the real Vitest: the changed test files run together give the scope that a run of each alone gives, and the -// same verdict when one of them cannot run. +// same verdict when one of them cannot run. Both scopes are taken in a child process, so that these runs are not +// part of what the mutation layer re-runs for every mutant of the diff step. const dirs: string[] = []; @@ -16,6 +17,21 @@ afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +const KIT = join(import.meta.dirname, "../.."); + +const SCRIPT = ` +const [kit, cwd, token] = process.argv.slice(2); +const { diffScope } = await import(kit + "/src/diff/scope.ts"); +const { vitestCoverageOf } = await import(kit + "/src/diff/coverage.ts"); +const { realExec } = await import(kit + "/src/gate/exec.ts"); +const alone = (file) => vitestCoverageOf(file, { cwd, exec: realExec }); +const copyToken = () => token; +const before = await diffScope({ copyToken, cwd, base: "main", coverageOf: alone }); +const after = await diffScope({ copyToken, cwd, base: "main" }); +const scope = after.scope; +console.log(JSON.stringify({ before, after, keys: scope ? scope.coveredFunctions.map((ref) => ref.key) : [], mutate: scope ? scope.mutate : [] })); +`; + const M = [ "export function f(a: number): number {", " return a + 1;", @@ -30,6 +46,8 @@ const M = [ ].join("\n"); const test = (calls: string): string => `import { expect, it } from "vitest";\nimport { f, g, h } from "./m.ts";\nit("runs", () => {\n expect(${calls}).toBeTypeOf("number");\n});\n`; +const BREAKS = + 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n'; function branch(files: Readonly>): string { const { project } = realProject({ ".gitignore": "node_modules\n.web-tops\n", ...files }); @@ -40,55 +58,63 @@ function branch(files: Readonly>): string { return project.dir; } +interface Scopes { + readonly before: unknown; + readonly after: unknown; + readonly keys: readonly unknown[]; + readonly mutate: readonly unknown[]; +} + +const listOf = (value: unknown): unknown[] => (Array.isArray(value) ? value : []); + +/** Both scopes of the project, as the child process took them: each file alone, and the files together. */ +function scopes(cwd: string): Scopes { + const dir = mkdtempSync(join(tmpdir(), "wt-real-batch-")); + dirs.push(dir); + const script = join(dir, "scopes.mjs"); + writeFileSync(script, SCRIPT); + const run = spawnSync("bun", [script, KIT, cwd, "tok"], { encoding: "utf8", cwd }); + expect(run.status, run.stderr).toBe(0); + const parsed: unknown = JSON.parse(run.stdout); + const field = (name: string): unknown => Reflect.get(Object(parsed), name); + return { + before: field("before"), + after: field("after"), + keys: listOf(field("keys")), + mutate: listOf(field("mutate")), + }; +} + describe("the real run of changed test files together", () => { it( "is the scope of a run of each alone: added, weakened and unchanged-source tests", - async () => { - const cwd = branch({ - "src/m.ts": M, - "src/weak.test.ts": test("f(1) + g(1)"), - }); + () => { + const cwd = branch({ "src/m.ts": M, "src/weak.test.ts": test("f(1) + g(1)") }); write(cwd, { "src/weak.test.ts": test("f(1)"), "src/a.test.ts": test("g(2)"), "src/b.test.ts": test("f(3)"), }); commit(cwd, "work"); - const alone = (file: string) => vitestCoverageOf(file, { cwd, exec: realExec }); - const before = await diffScope({ - copyToken: () => "tok", - cwd, - base: "main", - coverageOf: alone, - }); - const after = await diffScope({ copyToken: () => "tok", cwd, base: "main" }); + const { before, after, keys, mutate } = scopes(cwd); expect(after).toEqual(before); - expect(after.outcome).toBe("scope"); - if (after.outcome !== "scope") return; - expect(after.scope.coveredFunctions.map((ref) => ref.key)).toEqual([ - "src/m.ts::f", - "src/m.ts::g", - ]); - expect(after.scope.mutate).toEqual(["src/m.ts:1-6"]); + expect(keys).toEqual(["src/m.ts::f", "src/m.ts::g"]); + expect(mutate).toEqual(["src/m.ts:1-6"]); }, REAL_TOOL_TIMEOUT * 3, ); it( "fails as a run of each alone fails, naming the first file that cannot run", - async () => { + () => { const cwd = branch({ "src/m.ts": M }); write(cwd, { "src/a.test.ts": test("f(1)"), - "src/b.test.ts": - 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n', - "src/c.test.ts": - 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n', + "src/b.test.ts": BREAKS, + "src/c.test.ts": BREAKS, }); commit(cwd, "work"); - const alone = (file: string) => vitestCoverageOf(file, { cwd, exec: realExec }); - const before = await diffScope({ cwd, base: "main", coverageOf: alone }); - const after = await diffScope({ cwd, base: "main" }); + const { before, after } = scopes(cwd); expect(after).toEqual(before); expect(after).toEqual({ outcome: "fail", diff --git a/src/diff/test-support.ts b/src/diff/test-support.ts index 7478c18..44db909 100644 --- a/src/diff/test-support.ts +++ b/src/diff/test-support.ts @@ -1,3 +1,4 @@ +import type { Exec, ExecResult } from "../gate/types.ts"; import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -60,6 +61,83 @@ export function commit(dir: string, message: string): void { git(dir, "commit", "-q", "-m", message); } +/** A directory with the given files and no repository: for a run whose git is a fake. */ +export function plainDir(files: Readonly>): string { + const dir = mkdtempSync(join(tmpdir(), "wt-diff-")); + made.push(dir); + write(dir, files); + return dir; +} + +export interface Edit { + readonly path: string; + readonly kind: "added" | "modified"; + /** The new-side lines the edit touches; line 1 by default. */ + readonly range?: readonly [number, number]; +} + +/** A `git diff -U0` text for the edits: what the diff step would read from git. */ +export function diffOf(edits: readonly Edit[]): string { + const blocks = edits.map((edit) => { + const [start, end] = edit.range ?? [1, 1]; + const count = end - start + 1; + const old = edit.kind === "added" ? "/dev/null" : `a/${edit.path}`; + const hunk = + edit.kind === "added" + ? `@@ -0,0 +${String(start)},${String(count)} @@` + : `@@ -${String(start)},0 +${String(start)},${String(count)} @@`; + const added = Array.from({ length: count }, () => "+x"); + return [ + `diff --git a/${edit.path} b/${edit.path}`, + `--- ${old}`, + `+++ b/${edit.path}`, + hunk, + ...added, + ].join("\n"); + }); + return `${blocks.join("\n")}\n`; +} + +/** The merge base the fake git gives. */ +export const FAKE_BASE = "abc1234"; + +/** + * A git that says the project is a full checkout on a branch whose merge base is `FAKE_BASE`, and gives `diff` as the + * diff and `atBase` as the files at the merge base. It keeps the arguments of every call in `seen`. + */ +export function fakeGit( + diff: string, + atBase: Readonly> = {}, + seen: string[][] = [], +): Exec { + return (_command, args) => { + seen.push([...args]); + return Promise.resolve(answerOf(args, diff, atBase)); + }; +} + +const said = (stdout: string): ExecResult => ({ code: 0, stdout, stderr: "" }); + +const ANSWERS: Readonly> = { + "--is-shallow-repository": "false", + "--is-inside-work-tree": "true", +}; + +function showOf(args: readonly string[], atBase: Readonly>): ExecResult { + const text = atBase[args[1]?.slice(FAKE_BASE.length + 1) ?? ""]; + return text === undefined ? { code: 128, stdout: "", stderr: "no such path" } : said(text); +} + +function answerOf( + args: readonly string[], + diff: string, + atBase: Readonly>, +): ExecResult { + if (args[0] === "show") return showOf(args, atBase); + if (args.includes("diff")) return said(diff); + return said(args.map((arg) => ANSWERS[arg]).find((answer) => answer !== undefined) ?? FAKE_BASE); +} + export function cleanRepos(): void { for (const dir of made.splice(0)) rmSync(dir, { recursive: true, force: true }); } From 53fd26b40b234f8f6c3495c03ce5449a974ef7d1 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 03:37:39 -0500 Subject: [PATCH 7/8] Pin the fake git of the diff tests, and the order of the covered functions The helpers the diff tests take their git from are source files to the mutation layer, so what they say is pinned by their own test: the text of a diff, what the fake git answers, and the directory a plain project is made in. The covered functions are listed by file and line, and a run that took its own directory away is no failure. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/batch.test.ts | 20 ++++++- src/diff/coverage-many.test.ts | 11 +++- src/diff/fake-git.test.ts | 100 +++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 src/diff/fake-git.test.ts diff --git a/src/diff/batch.test.ts b/src/diff/batch.test.ts index 967262f..10e2f24 100644 --- a/src/diff/batch.test.ts +++ b/src/diff/batch.test.ts @@ -5,7 +5,7 @@ import type { Exec } from "../gate/types.ts"; import type { CoverageOf, CoverageOfMany } from "./scope.ts"; import { diffScope } from "./scope.ts"; import { cleanRepos, diffOf, fakeGit, plainDir } from "./test-support.ts"; -import type { CoverageMap, ScopeResult } from "./types.ts"; +import type { CoverageMap, FunctionHit, ScopeResult } from "./types.ts"; // The changed test files are run together, once, and file by file only where that gives nothing: the functions in // scope, the mutate ranges and the verdicts are those of a run of each file alone. @@ -202,6 +202,24 @@ describe("changed test files run together", () => { expect(sizes).toEqual([100, 100, 5]); }); + it("lists the functions by file and line, whatever order the coverage came in", async () => { + const cwd = plainDir({ "src/z.ts": M, "src/a.ts": M, "src/t.test.ts": WEAK }); + const both = (path: string): [string, readonly FunctionHit[]] => [ + path, + [ + { start: 4, end: 6, hits: 1 }, + { start: 1, end: 3, hits: 1 }, + ], + ]; + const result = await diffScope({ + cwd, + base: "main", + exec: fakeGit(diffOf([{ path: "src/t.test.ts", kind: "added" }])), + coverageOfMany: () => Promise.resolve(new Map([both("src/z.ts"), both("src/a.ts")])), + }); + expect(keys(result)).toEqual(["src/a.ts::f", "src/a.ts::g", "src/z.ts::f", "src/z.ts::g"]); + }); + it("runs no group when it is given a source for one file only", async () => { const asked: string[] = []; const project = added("src/a.test.ts", "src/b.test.ts"); diff --git a/src/diff/coverage-many.test.ts b/src/diff/coverage-many.test.ts index 8267438..ad76a45 100644 --- a/src/diff/coverage-many.test.ts +++ b/src/diff/coverage-many.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import type { Exec } from "../gate/types.ts"; @@ -60,6 +60,15 @@ describe("vitestCoverageOfMany", () => { expect(extra).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length - 1 + 2); }); + it("is undefined, not a failure, when the run took its own directory away", async () => { + const exec: Exec = (_command, args) => { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + rmSync(flag.slice("--coverage.reportsDirectory=".length), { recursive: true, force: true }); + return Promise.resolve({ code: 1, stdout: "", stderr: "" }); + }; + expect(await vitestCoverageOfMany(["src/a.test.ts"], { cwd: "/repo", exec })).toBeUndefined(); + }); + it("is undefined when the run fails, whichever file is the cause", async () => { const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { cwd: "/repo", diff --git a/src/diff/fake-git.test.ts b/src/diff/fake-git.test.ts new file mode 100644 index 0000000..7afae0e --- /dev/null +++ b/src/diff/fake-git.test.ts @@ -0,0 +1,100 @@ +import { existsSync, readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { cleanRepos, diffOf, FAKE_BASE, fakeGit, plainDir } from "./test-support.ts"; + +// The helpers that the diff tests take their git from, pinned so that what they say is what git would say. + +const run = (git: ReturnType, ...args: string[]) => git("git", args, { cwd: "/x" }); + +describe("plainDir", () => { + it("is a directory with the files, named for the diff tests, that cleanRepos removes", () => { + const dir = plainDir({ "src/a.ts": "export {};\n" }); + expect(basename(dir).startsWith("wt-diff-")).toBe(true); + expect(readFileSync(join(dir, "src/a.ts"), "utf8")).toBe("export {};\n"); + cleanRepos(); + expect(existsSync(dir)).toBe(false); + }); +}); + +describe("diffOf", () => { + it("writes an added file as a -U0 diff from /dev/null, over the lines it gives", () => { + expect(diffOf([{ path: "src/a.ts", kind: "added", range: [1, 2] }])).toBe( + [ + "diff --git a/src/a.ts b/src/a.ts", + "--- /dev/null", + "+++ b/src/a.ts", + "@@ -0,0 +1,2 @@", + "+x", + "+x", + "", + ].join("\n"), + ); + }); + + it("writes a modified file against itself, at the lines it gives, and one line 1 by default", () => { + expect(diffOf([{ path: "src/a.ts", kind: "modified", range: [3, 5] }])).toBe( + [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + "@@ -3,0 +3,3 @@", + "+x", + "+x", + "+x", + "", + ].join("\n"), + ); + expect(diffOf([{ path: "b.ts", kind: "modified" }])).toContain("@@ -1,0 +1,1 @@"); + }); + + it("puts the files one after another", () => { + const text = diffOf([ + { path: "a.ts", kind: "added" }, + { path: "b.ts", kind: "modified" }, + ]); + expect(text.split("\n").filter((line) => line.startsWith("diff --git"))).toEqual([ + "diff --git a/a.ts b/a.ts", + "diff --git a/b.ts b/b.ts", + ]); + }); +}); + +describe("fakeGit", () => { + it("says a full checkout whose merge base is FAKE_BASE", async () => { + const git = fakeGit(""); + expect((await run(git, "rev-parse", "--is-inside-work-tree")).stdout).toBe("true"); + expect((await run(git, "rev-parse", "--is-shallow-repository")).stdout).toBe("false"); + expect((await run(git, "rev-parse", "--verify", "--quiet", "main^{commit}")).stdout).toBe( + FAKE_BASE, + ); + expect((await run(git, "merge-base", "HEAD", "main")).stdout).toBe(FAKE_BASE); + }); + + it("gives the diff it was given", async () => { + expect((await run(fakeGit("the diff"), "-c", "x=y", "diff", "--", "ref")).stdout).toBe( + "the diff", + ); + }); + + it("gives a file at the merge base, and a failure for a file that was not there", async () => { + const git = fakeGit("", { "src/a.ts": "text" }); + expect(await run(git, "show", `${FAKE_BASE}:src/a.ts`)).toEqual({ + code: 0, + stdout: "text", + stderr: "", + }); + expect(await run(git, "show", `${FAKE_BASE}:src/b.ts`)).toEqual({ + code: 128, + stdout: "", + stderr: "no such path", + }); + expect((await run(git, "show")).code).toBe(128); + }); + + it("keeps the arguments of every call", async () => { + const seen: string[][] = []; + await run(fakeGit("", {}, seen), "merge-base", "HEAD", "main"); + expect(seen).toEqual([["merge-base", "HEAD", "main"]]); + }); +}); From 06765870556a4a8f969d49ac93608e77e3119e92 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 03:52:02 -0500 Subject: [PATCH 8/8] Pin the order of one file's covered functions, and ledger two equivalent mutants of the test helpers Co-Authored-By: Claude Sonnet 5.5 --- .web-tops/survivors.yaml | 20 ++++++++++++++++++++ src/diff/batch.test.ts | 15 +++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/.web-tops/survivors.yaml b/.web-tops/survivors.yaml index e77e35f..d8ea337 100644 --- a/.web-tops/survivors.yaml +++ b/.web-tops/survivors.yaml @@ -174,6 +174,26 @@ survivors: function to the changed ones, whether or not it is counted among the sources. owner: web-platform + - file: src/diff/test-support.ts + function: src/diff/test-support.ts:: + mutator: ArrayDeclaration + original: "[]" + replacement: "[\"Stryker was here\"]" + occurrence: 1 + reason: The list only holds directories to remove; a name that is no directory + is removed with force and changes nothing, so a first entry in the list is + not seen. + owner: web-platform + - file: src/diff/test-support.ts + function: src/diff/test-support.ts::showOf + mutator: StringLiteral + original: "\"\"" + replacement: "\"Stryker was here!\"" + occurrence: 1 + reason: "The fallback is the key of a file that is not in the table of + merge-base files, and any other string is not in it either: the same + failure comes back." + owner: web-platform - file: src/gate/adapter.ts function: src/gate/adapter.ts::classCap mutator: StringLiteral diff --git a/src/diff/batch.test.ts b/src/diff/batch.test.ts index 10e2f24..eabba29 100644 --- a/src/diff/batch.test.ts +++ b/src/diff/batch.test.ts @@ -202,6 +202,21 @@ describe("changed test files run together", () => { expect(sizes).toEqual([100, 100, 5]); }); + it("lists the functions of one file by line, though the head covered the later and the base the earlier", async () => { + const project = weakened(); + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOf: () => Promise.resolve(undefined), + coverageOfMany: (group) => + Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? ran(true, false) : ran(false, true), + ), + }); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + it("lists the functions by file and line, whatever order the coverage came in", async () => { const cwd = plainDir({ "src/z.ts": M, "src/a.ts": M, "src/t.test.ts": WEAK }); const both = (path: string): [string, readonly FunctionHit[]] => [