diff --git a/.web-tops/survivors.yaml b/.web-tops/survivors.yaml index c73d8dc..d8ea337 100644 --- a/.web-tops/survivors.yaml +++ b/.web-tops/survivors.yaml @@ -135,6 +135,65 @@ survivors: reason: document.toJS throws only Error instances (the YAML library), so the fallback text is a dead branch. owner: web-platform + - file: src/diff/coverage.ts + function: src/diff/coverage.ts::innermostHolding + mutator: ConditionalExpression + original: "span.start <= hit.start" + replacement: "true" + occurrence: 1 + reason: innermostAt then keeps only spans that hold the line of the hit's start, + which are exactly the spans that start at or before it, so the first test + of the filter changes nothing. + owner: web-platform + - file: src/diff/functions.ts + function: src/diff/functions.ts::boundName + mutator: ConditionalExpression + original: "parent.initializer === node" + replacement: "true" + occurrence: 1 + reason: "A function node is the child of a variable, property assignment or + property declaration only as its initializer: its other children are + names, types and decorators, none of them function nodes." + owner: web-platform + - file: src/diff/functions.ts + function: src/diff/functions.ts::declaredName + mutator: ConditionalExpression + original: "ts.isArrowFunction(node)" + replacement: "false" + occurrence: 1 + reason: An arrow function has no name, so the next line gives undefined for it + as well; the test exists for the type of the node, not for the result. + owner: web-platform + - file: src/diff/scope.ts + function: src/diff/scope.ts::scopeOf + mutator: ConditionalExpression + original: "file.status !== \"deleted\"" + replacement: "true" + occurrence: 1 + reason: A deleted file has no added lines, so it adds no range to mutate and no + function to the changed ones, whether or not it is counted among the + sources. + owner: web-platform + - file: src/diff/test-support.ts + function: src/diff/test-support.ts:: + mutator: ArrayDeclaration + original: "[]" + replacement: "[\"Stryker was here\"]" + occurrence: 1 + reason: The list only holds directories to remove; a name that is no directory + is removed with force and changes nothing, so a first entry in the list is + not seen. + owner: web-platform + - file: src/diff/test-support.ts + function: src/diff/test-support.ts::showOf + mutator: StringLiteral + original: "\"\"" + replacement: "\"Stryker was here!\"" + occurrence: 1 + reason: "The fallback is the key of a file that is not in the table of + merge-base files, and any other string is not in it either: the same + failure comes back." + owner: web-platform - file: src/gate/adapter.ts function: src/gate/adapter.ts::classCap mutator: StringLiteral diff --git a/CHANGELOG.md b/CHANGELOG.md index 97ab4b4..333a4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project are listed here. The protocol version and th ## [Unreleased] - Milestone M3 opens: `knip`, `secretlint` with its recommended preset and `eslint-plugin-react-hooks` are pinned exactly in §2, `package.json` and the lock, and the M3 paths are in every tool scope. §8.1 and §17 name the law fixtures `fixtures/laws/W-NN/{bad,clean}/`. (G0) +- The diff step finds what the changed test files cover with one Vitest run for their HEAD versions and one for their merge-base versions, not one run of each file alone, and runs file by file only where a group gives nothing. The functions in scope, the mutants and the verdicts are the same; the cost of the step falls with the number of changed test files. ## [0.3.0] - 2026-11-19 diff --git a/WEB_TOPS.md b/WEB_TOPS.md index 6a75da5..4784604 100644 --- a/WEB_TOPS.md +++ b/WEB_TOPS.md @@ -390,10 +390,11 @@ banned shortcuts, `!` and `as`, cheaper than handling the case. The profile key test file at the merge base or at HEAD**, so a change that only weakens a test, or drops the call to a function, is mutated. `webtops` computes the line ranges and passes them as Stryker `mutate` entries, `file:start-end` (confirmed on Stryker 10.1.1). It maps each changed test file to the - functions it covers by running that file alone under Vitest, with the gate's own configuration + functions it covers by running the changed test files under Vitest, with the gate's own configuration (the project's, coverage thresholds removed) and the gate's coverage include and exclude (§6.1), - once for the version at HEAD and once for the version at the merge base; a deleted test file runs - only the second. The functions that ran are added, with their whole ranges. Each range is then widened to the scored units it reaches into (§6.7), and a range that touches module-level code to all of the file's module-level lines, so the occurrence in a survivor's key does not depend on the diff. A HEAD version that + once for the versions at HEAD and once for the versions at the merge base, each in one run of up to 100 files (a function + is covered when any file ran it; a group whose run gives nothing runs file by file, so the file that cannot run is + found); a deleted test file is in the second only. The functions that ran are added, with their whole ranges. Each range is then widened to the scored units it reaches into (§6.7), and a range that touches module-level code to all of the file's module-level lines, so the occurrence in a survivor's key does not depend on the diff. A HEAD version that cannot run is `fail`; a merge-base version that cannot run against HEAD adds nothing. The diff is taken without rename detection, so a renamed file counts as new and every function in it is judged. A pure deletion belongs to the smallest function that holds the lines on both sides of diff --git a/src/diff/batch.test.ts b/src/diff/batch.test.ts new file mode 100644 index 0000000..eabba29 --- /dev/null +++ b/src/diff/batch.test.ts @@ -0,0 +1,254 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Exec } from "../gate/types.ts"; +import type { CoverageOf, CoverageOfMany } from "./scope.ts"; +import { diffScope } from "./scope.ts"; +import { cleanRepos, diffOf, fakeGit, plainDir } from "./test-support.ts"; +import type { CoverageMap, FunctionHit, ScopeResult } from "./types.ts"; + +// The changed test files are run together, once, and file by file only where that gives nothing: the functions in +// scope, the mutate ranges and the verdicts are those of a run of each file alone. + +afterEach(cleanRepos); + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "", +].join("\n"); +const STRONG = 'import { f, g } from "./m.ts";\nf(1);\ng(2);\n'; +const WEAK = 'import { f } from "./m.ts";\nf(1);\n'; + +/** What a run covered of src/m.ts: f at lines 1 to 3, g at 4 to 6. */ +const ran = (f: boolean, g: boolean): CoverageMap => + new Map([ + [ + "src/m.ts", + [ + { start: 1, end: 3, hits: f ? 1 : 0 }, + { start: 4, end: 6, hits: g ? 1 : 0 }, + ], + ], + ]); + +/** A project with src/m.ts and the given test files added, and the git that would say so. */ +function added(...names: string[]): { cwd: string; exec: Exec } { + const cwd = plainDir({ "src/m.ts": M, ...Object.fromEntries(names.map((name) => [name, WEAK])) }); + const exec = fakeGit(diffOf(names.map((path) => ({ path, kind: "added" })))); + return { cwd, exec }; +} + +/** src/m.test.ts weakened and src/n.test.ts added, with the strong version of the first at the merge base. */ +function weakened(): { cwd: string; exec: Exec } { + const cwd = plainDir({ "src/m.ts": M, "src/m.test.ts": WEAK, "src/n.test.ts": WEAK }); + const diff = diffOf([ + { path: "src/m.test.ts", kind: "modified" }, + { path: "src/n.test.ts", kind: "added" }, + ]); + return { cwd, exec: fakeGit(diff, { "src/m.test.ts": STRONG }) }; +} + +const keys = (result: ScopeResult): string[] => + result.outcome === "fail" ? [] : result.scope.coveredFunctions.map((ref) => ref.key); + +describe("changed test files run together", () => { + it("runs the head versions in one run, and none of them alone", async () => { + const together: (readonly string[])[] = []; + const alone: string[] = []; + const result = await diffScope({ + ...added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + base: "main", + coverageOfMany: (group) => { + together.push(group); + return Promise.resolve(ran(true, true)); + }, + coverageOf: (file) => { + alone.push(file); + return Promise.resolve(undefined); + }, + }); + expect(together).toEqual([["src/a.test.ts", "src/b.test.ts", "src/c.test.ts"]]); + expect(alone).toEqual([]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + + it("runs them one by one, in order, when the run together gives nothing, and adds up what each covered", async () => { + const alone: string[] = []; + const result = await diffScope({ + ...added("src/a.test.ts", "src/b.test.ts"), + base: "main", + coverageOfMany: () => Promise.resolve(undefined), + coverageOf: (file) => { + alone.push(file); + return Promise.resolve(file === "src/a.test.ts" ? ran(true, false) : ran(false, true)); + }, + }); + expect(alone).toEqual(["src/a.test.ts", "src/b.test.ts"]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + + it("fails naming the first file that cannot run alone, the later ones unnamed", async () => { + const result = await diffScope({ + ...added("src/a.test.ts", "src/b.test.ts", "src/c.test.ts"), + base: "main", + coverageOfMany: () => Promise.resolve(undefined), + coverageOf: (file) => Promise.resolve(file === "src/a.test.ts" ? ran(true, true) : undefined), + }); + expect(result).toEqual({ + outcome: "fail", + reason: "could not run src/b.test.ts to find the functions it covers", + }); + }); + + it("is the same scope as running each file alone", async () => { + const project = added("src/a.test.ts", "src/b.test.ts"); + const maps: Record = { + "src/a.test.ts": ran(true, false), + "src/b.test.ts": ran(false, true), + }; + const alone: CoverageOf = (file) => Promise.resolve(maps[file]); + const many: CoverageOfMany = () => Promise.resolve(ran(true, true)); + const one = await diffScope({ ...project, base: "main", coverageOf: alone }); + const all = await diffScope({ + ...project, + base: "main", + coverageOfMany: many, + coverageOf: alone, + }); + expect(all).toEqual(one); + }); + + it("runs the merge-base versions together too, as copies that are gone afterwards", async () => { + const project = weakened(); + const together: (readonly string[])[] = []; + const seen: string[] = []; + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOf: () => Promise.resolve(undefined), + coverageOfMany: (group) => { + together.push(group); + for (const file of group) seen.push(existsSync(join(project.cwd, file)) ? file : ""); + return Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? ran(true, true) : ran(true, false), + ); + }, + }); + expect(together).toEqual([["src/m.test.ts", "src/n.test.ts"], ["src/m.wt-base-tok.test.ts"]]); + expect(seen).toEqual(["src/m.test.ts", "src/n.test.ts", "src/m.wt-base-tok.test.ts"]); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("holds the merge-base version in the copy, and runs a copy that fails together alone, leaving it out", async () => { + const project = weakened(); + const alone: string[] = []; + const text: Record = {}; + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOfMany: (group) => + Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? undefined : ran(true, false), + ), + coverageOf: (file) => { + alone.push(file); + text[file] = readFileSync(join(project.cwd, file), "utf8"); + return Promise.resolve(undefined); + }, + }); + expect(alone).toEqual(["src/m.wt-base-tok.test.ts"]); + expect(text["src/m.wt-base-tok.test.ts"]).toBe(STRONG.trim()); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("leaves a merge-base version out when its runs throw, and still removes the copy", async () => { + const project = weakened(); + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOfMany: (group) => { + if (group.some((file) => file.includes(".wt-base-"))) throw new Error("the run died"); + return Promise.resolve(ran(true, false)); + }, + coverageOf: () => { + throw new Error("the run died"); + }, + }); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(existsSync(join(project.cwd, "src/m.wt-base-tok.test.ts"))).toBe(false); + }); + + it("runs at most 100 files at once", async () => { + const names = Array.from({ length: 205 }, (_, at) => `src/t${String(at)}.test.ts`); + const sizes: number[] = []; + await diffScope({ + ...added(...names), + base: "main", + coverageOfMany: (group) => { + sizes.push(group.length); + return Promise.resolve(ran(true, false)); + }, + }); + expect(sizes).toEqual([100, 100, 5]); + }); + + it("lists the functions of one file by line, though the head covered the later and the base the earlier", async () => { + const project = weakened(); + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOf: () => Promise.resolve(undefined), + coverageOfMany: (group) => + Promise.resolve( + group.some((file) => file.includes(".wt-base-")) ? ran(true, false) : ran(false, true), + ), + }); + expect(keys(result)).toEqual(["src/m.ts::f", "src/m.ts::g"]); + }); + + it("lists the functions by file and line, whatever order the coverage came in", async () => { + const cwd = plainDir({ "src/z.ts": M, "src/a.ts": M, "src/t.test.ts": WEAK }); + const both = (path: string): [string, readonly FunctionHit[]] => [ + path, + [ + { start: 4, end: 6, hits: 1 }, + { start: 1, end: 3, hits: 1 }, + ], + ]; + const result = await diffScope({ + cwd, + base: "main", + exec: fakeGit(diffOf([{ path: "src/t.test.ts", kind: "added" }])), + coverageOfMany: () => Promise.resolve(new Map([both("src/z.ts"), both("src/a.ts")])), + }); + expect(keys(result)).toEqual(["src/a.ts::f", "src/a.ts::g", "src/z.ts::f", "src/z.ts::g"]); + }); + + it("runs no group when it is given a source for one file only", async () => { + const asked: string[] = []; + const project = added("src/a.test.ts", "src/b.test.ts"); + const exec: Exec = (command, args, options) => { + asked.push(command); + return project.exec(command, args, options); + }; + const result = await diffScope({ + cwd: project.cwd, + base: "main", + exec, + coverageOf: () => Promise.resolve(ran(true, false)), + }); + expect(keys(result)).toEqual(["src/m.ts::f"]); + expect(asked.filter((command) => command.includes("vitest"))).toEqual([]); + }); +}); diff --git a/src/diff/coverage-many.test.ts b/src/diff/coverage-many.test.ts new file mode 100644 index 0000000..ad76a45 --- /dev/null +++ b/src/diff/coverage-many.test.ts @@ -0,0 +1,86 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { Exec } from "../gate/types.ts"; +import { DEFAULT_COVERAGE_EXCLUDES } from "../layers/coverage/scope.ts"; +import { vitestCoverageOf, vitestCoverageOfMany } from "./coverage.ts"; + +const REPORT = { + "/repo/src/a.ts": { + fnMap: { 0: { loc: { start: { line: 1 }, end: { line: 3 } } } }, + f: { 0: 1 }, + }, +}; + +/** An exec that writes the report for a run that exits 0, and keeps the arguments of every run. */ +function stamped(log: string[][], code = 0): Exec { + return (command, args) => { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")); + const out = flag?.slice("--coverage.reportsDirectory=".length) ?? ""; + mkdirSync(out, { recursive: true }); + writeFileSync(join(out, "coverage-final.json"), JSON.stringify(REPORT)); + log.push([command, ...args]); + return Promise.resolve({ code, stdout: "", stderr: "" }); + }; +} + +describe("vitestCoverageOfMany", () => { + it("runs Vitest once on every file it is given, and reads one map", async () => { + const log: string[][] = []; + const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped(log), + }); + expect(log).toHaveLength(1); + const args = log[0] ?? []; + expect(args.slice(1, 4)).toEqual(["run", "src/a.test.ts", "src/b.test.ts"]); + expect(map?.get("src/a.ts")).toEqual([{ start: 1, end: 3, hits: 1 }]); + }); + + it("gives the gate's own options and nothing else, then the extra arguments it is given", async () => { + const log: string[][] = []; + await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped(log), + }); + const plain = (log[0] ?? []).slice(1); + expect(plain).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length); + expect(plain.filter((arg) => arg.startsWith("--coverage.exclude="))).toEqual( + DEFAULT_COVERAGE_EXCLUDES.map((glob) => `--coverage.exclude=${glob}`), + ); + await vitestCoverageOfMany(["src/a.test.ts"], { + cwd: "/repo", + exec: stamped(log), + exclude: ["gen/**"], + args: ["--reporter=dot"], + }); + const extra = (log[1] ?? []).slice(1); + expect(extra.at(-1)).toBe("--reporter=dot"); + expect(extra).toContain("--coverage.exclude=gen/**"); + expect(extra).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length - 1 + 2); + }); + + it("is undefined, not a failure, when the run took its own directory away", async () => { + const exec: Exec = (_command, args) => { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + rmSync(flag.slice("--coverage.reportsDirectory=".length), { recursive: true, force: true }); + return Promise.resolve({ code: 1, stdout: "", stderr: "" }); + }; + expect(await vitestCoverageOfMany(["src/a.test.ts"], { cwd: "/repo", exec })).toBeUndefined(); + }); + + it("is undefined when the run fails, whichever file is the cause", async () => { + const map = await vitestCoverageOfMany(["src/a.test.ts", "src/b.test.ts"], { + cwd: "/repo", + exec: stamped([], 1), + }); + expect(map).toBeUndefined(); + }); + + it("is, for one file, the run of that file alone", async () => { + const log: string[][] = []; + await vitestCoverageOf("src/a.test.ts", { cwd: "/repo", exec: stamped(log) }); + expect((log[0] ?? []).slice(1, 3)).toEqual(["run", "src/a.test.ts"]); + expect((log[0] ?? []).filter((arg) => arg.endsWith(".test.ts"))).toEqual(["src/a.test.ts"]); + }); +}); diff --git a/src/diff/coverage.ts b/src/diff/coverage.ts index 0bdb89f..495436a 100644 --- a/src/diff/coverage.ts +++ b/src/diff/coverage.ts @@ -66,11 +66,9 @@ function hitsByFile( */ export function parseCoverage( value: unknown, - root?: string | readonly string[], + root: string | readonly string[] = [], ): CoverageMap | undefined { - return isRecord(value) - ? hitsByFile(value, typeof root === "string" ? [root] : (root ?? [])) - : undefined; + return isRecord(value) ? hitsByFile(value, typeof root === "string" ? [root] : root) : undefined; } /** The functions, of those the spans list, that ran: each hit goes to the smallest function-or-module span that holds it. */ @@ -101,11 +99,16 @@ export interface CoverageRun { readonly args?: readonly string[]; } -function coverageArgs(testFile: string, config: string, out: string, run: CoverageRun): string[] { +function coverageArgs( + testFiles: readonly string[], + config: string, + out: string, + run: CoverageRun, +): string[] { const excludes = [...DEFAULT_COVERAGE_EXCLUDES, ...(run.exclude ?? [])]; return [ "run", - testFile, + ...testFiles, `--config=${config}`, "--coverage", "--coverage.provider=v8", @@ -118,18 +121,19 @@ function coverageArgs(testFile: string, config: string, out: string, run: Covera } /** - * Runs Vitest on one test file with the gate's own config (the project's, without coverage thresholds) and - * the gate's coverage include and exclude, as the tests layer does, and reads what ran from the JSON report. - * A run that fails, or leaves no coverage file, is undefined: what the test covers is then not known. + * Runs Vitest on test files, all in one run, with the gate's own config (the project's, without coverage thresholds) + * and the gate's coverage include and exclude, as the tests layer does, and reads what ran from the JSON report: a + * function is covered when any of the files ran it. A run that fails, or leaves no coverage file, is undefined: what + * the files cover is then not known, and which of them is the cause is not either. */ -export async function vitestCoverageOf( - testFile: string, +export async function vitestCoverageOfMany( + testFiles: readonly string[], run: CoverageRun, ): Promise { const out = mkdtempSync(join(tmpdir(), "wt-cov-")); try { const config = writeKitConfig(run.cwd, out); - const args = coverageArgs(testFile, config, out, run); + const args = coverageArgs(testFiles, config, out, run); const result = await run.exec(join(run.cwd, "node_modules/.bin/vitest"), args, { cwd: run.cwd, }); @@ -139,6 +143,12 @@ export async function vitestCoverageOf( } } +/** What one test file covers: a run of that file alone. */ +export const vitestCoverageOf = ( + testFile: string, + run: CoverageRun, +): Promise => vitestCoverageOfMany([testFile], run); + /** The work tree under its given name and under its real path, when it has one. */ function rootsOf(root: string): readonly string[] { try { diff --git a/src/diff/fake-git.test.ts b/src/diff/fake-git.test.ts new file mode 100644 index 0000000..7afae0e --- /dev/null +++ b/src/diff/fake-git.test.ts @@ -0,0 +1,100 @@ +import { existsSync, readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { cleanRepos, diffOf, FAKE_BASE, fakeGit, plainDir } from "./test-support.ts"; + +// The helpers that the diff tests take their git from, pinned so that what they say is what git would say. + +const run = (git: ReturnType, ...args: string[]) => git("git", args, { cwd: "/x" }); + +describe("plainDir", () => { + it("is a directory with the files, named for the diff tests, that cleanRepos removes", () => { + const dir = plainDir({ "src/a.ts": "export {};\n" }); + expect(basename(dir).startsWith("wt-diff-")).toBe(true); + expect(readFileSync(join(dir, "src/a.ts"), "utf8")).toBe("export {};\n"); + cleanRepos(); + expect(existsSync(dir)).toBe(false); + }); +}); + +describe("diffOf", () => { + it("writes an added file as a -U0 diff from /dev/null, over the lines it gives", () => { + expect(diffOf([{ path: "src/a.ts", kind: "added", range: [1, 2] }])).toBe( + [ + "diff --git a/src/a.ts b/src/a.ts", + "--- /dev/null", + "+++ b/src/a.ts", + "@@ -0,0 +1,2 @@", + "+x", + "+x", + "", + ].join("\n"), + ); + }); + + it("writes a modified file against itself, at the lines it gives, and one line 1 by default", () => { + expect(diffOf([{ path: "src/a.ts", kind: "modified", range: [3, 5] }])).toBe( + [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + "@@ -3,0 +3,3 @@", + "+x", + "+x", + "+x", + "", + ].join("\n"), + ); + expect(diffOf([{ path: "b.ts", kind: "modified" }])).toContain("@@ -1,0 +1,1 @@"); + }); + + it("puts the files one after another", () => { + const text = diffOf([ + { path: "a.ts", kind: "added" }, + { path: "b.ts", kind: "modified" }, + ]); + expect(text.split("\n").filter((line) => line.startsWith("diff --git"))).toEqual([ + "diff --git a/a.ts b/a.ts", + "diff --git a/b.ts b/b.ts", + ]); + }); +}); + +describe("fakeGit", () => { + it("says a full checkout whose merge base is FAKE_BASE", async () => { + const git = fakeGit(""); + expect((await run(git, "rev-parse", "--is-inside-work-tree")).stdout).toBe("true"); + expect((await run(git, "rev-parse", "--is-shallow-repository")).stdout).toBe("false"); + expect((await run(git, "rev-parse", "--verify", "--quiet", "main^{commit}")).stdout).toBe( + FAKE_BASE, + ); + expect((await run(git, "merge-base", "HEAD", "main")).stdout).toBe(FAKE_BASE); + }); + + it("gives the diff it was given", async () => { + expect((await run(fakeGit("the diff"), "-c", "x=y", "diff", "--", "ref")).stdout).toBe( + "the diff", + ); + }); + + it("gives a file at the merge base, and a failure for a file that was not there", async () => { + const git = fakeGit("", { "src/a.ts": "text" }); + expect(await run(git, "show", `${FAKE_BASE}:src/a.ts`)).toEqual({ + code: 0, + stdout: "text", + stderr: "", + }); + expect(await run(git, "show", `${FAKE_BASE}:src/b.ts`)).toEqual({ + code: 128, + stdout: "", + stderr: "no such path", + }); + expect((await run(git, "show")).code).toBe(128); + }); + + it("keeps the arguments of every call", async () => { + const seen: string[][] = []; + await run(fakeGit("", {}, seen), "merge-base", "HEAD", "main"); + expect(seen).toEqual([["merge-base", "HEAD", "main"]]); + }); +}); diff --git a/src/diff/functions.ts b/src/diff/functions.ts index 8fa34e4..e092337 100644 --- a/src/diff/functions.ts +++ b/src/diff/functions.ts @@ -107,7 +107,7 @@ function kindOf(file: string): ts.ScriptKind { export function functionsIn(file: string, text: string): readonly FunctionSpan[] { const source = ts.createSourceFile(file, text, ts.ScriptTarget.ES2023, true, kindOf(file)); const module = `${file}::`; - const last = text.replace(/\n$/, "").split("\n").length; + const last = text.split("\n").length - (text.endsWith("\n") ? 1 : 0); const walk: Walk = { file, source, diff --git a/src/diff/pins.test.ts b/src/diff/pins.test.ts new file mode 100644 index 0000000..3671929 --- /dev/null +++ b/src/diff/pins.test.ts @@ -0,0 +1,323 @@ +import { existsSync, rmSync } from "node:fs"; +import { basename, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Exec } from "../gate/types.ts"; +import { COVERAGE_INCLUDE, DEFAULT_COVERAGE_EXCLUDES } from "../layers/coverage/scope.ts"; +import { coveredSpans, parseCoverage } from "./coverage.ts"; +import { functionsIn, innermostAt } from "./functions.ts"; +import { mergeBaseOf } from "./git.ts"; +import { diffScope, isSourceFile, isTestFile } from "./scope.ts"; +import { cleanRepos, diffOf, FAKE_BASE, fakeGit, plainDir, write } from "./test-support.ts"; +import type { CoverageMap } from "./types.ts"; + +// What the diff step does at its edges, pinned one fact at a time. + +afterEach(cleanRepos); + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "", +].join("\n"); +const TEST = 'import { f, g } from "./m.ts";\nf(1);\ng(2);\n'; + +const ran = (f: boolean, g: boolean): CoverageMap => + new Map([ + [ + "src/m.ts", + [ + { start: 1, end: 3, hits: f ? 1 : 0 }, + { start: 4, end: 6, hits: g ? 1 : 0 }, + ], + ], + ]); + +describe("which paths are tests and which are source", () => { + it.each([ + "a.test.ts", + "a.spec.ts", + "a.test.tsx", + "a.test.mts", + "a.spec.cts", + "src/__tests__/a.ts", + "__tests__/a.ts", + ])("%s is a test", (path) => { + expect(isTestFile(path)).toBe(true); + expect(isSourceFile(path)).toBe(false); + }); + + it.each(["a.ts", "a.tsx", "a.mts", "a.cts", "a.test.tsy.ts", "a.d.ts.ts", "src/__tests__x/a.ts"])( + "%s is source", + (path) => { + expect(isSourceFile(path)).toBe(true); + expect(isTestFile(path)).toBe(false); + }, + ); + + it.each(["a.js", "a.ts.bak", "a.test.js", "src/__tests__/a.js", "a.d.ts", "a.d.mts", "a.d.cts"])( + "%s is neither", + (path) => { + expect(isSourceFile(path)).toBe(false); + expect(isTestFile(path)).toBe(false); + }, + ); +}); + +describe("coverage maps", () => { + it("refuses a file whose functions have no location, and keeps one that has them", () => { + const entry = { loc: { start: { line: 1 }, end: { line: 2 } } }; + expect(parseCoverage({ "a.ts": { fnMap: { 0: {} }, f: { 0: 1 } } })).toBeUndefined(); + expect(parseCoverage({ "a.ts": { fnMap: { 0: entry }, f: { 0: 1 } } })?.get("a.ts")).toEqual([ + { start: 1, end: 2, hits: 1 }, + ]); + }); + + it("takes no work-tree prefix off a path when it is given none", () => { + const file = { fnMap: { 0: { loc: { start: { line: 1 }, end: { line: 2 } } } }, f: { 0: 1 } }; + const paths = (path: string): string[] => [...(parseCoverage({ [path]: file })?.keys() ?? [])]; + expect(paths("/work/a.ts")).toEqual(["/work/a.ts"]); + expect(paths("Stryker was here/a.ts")).toEqual(["Stryker was here/a.ts"]); + }); + + const spans = [ + { key: "a.ts::", start: 1, end: 20 }, + { key: "a.ts::f", start: 3, end: 10 }, + { key: "a.ts::g", start: 12, end: 15 }, + ]; + + it("never lists the module as a covered function, whatever ran in it", () => { + expect(coveredSpans(spans, [{ start: 1, end: 20, hits: 3 }])).toEqual([]); + }); + + it("gives a hit to a function only when the function holds all of it", () => { + expect(coveredSpans(spans, [{ start: 5, end: 11, hits: 1 }])).toEqual([]); + expect(coveredSpans(spans, [{ start: 5, end: 8, hits: 1 }]).map((span) => span.key)).toEqual([ + "a.ts::f", + ]); + }); +}); + +describe("functions of a file", () => { + it("leaves out what has no body: overloads, abstract members and signatures", () => { + const text = [ + "export function ov(a: string): void;", + "export function ov(a: number): void;", + "export function ov(a: string | number): void {}", + "abstract class A {", + " abstract m(): void;", + "}", + "interface I {", + " n(): void;", + "}", + "namespace N {", + " export const x = 1;", + "}", + "", + ].join("\n"); + expect(functionsIn("a.ts", text).map((span) => span.key)).toEqual([ + "a.ts::", + "a.ts::ov", + ]); + }); + + it("names a function only by a variable, property assignment or property declaration it initializes", () => { + const text = "export function f(a = () => 1) {}\nconst { x = () => 2 } = o;\n"; + expect(functionsIn("a.ts", text).map((span) => span.key)).toEqual([ + "a.ts::", + "a.ts::f", + "a.ts::f#1", + "a.ts::#1", + ]); + }); + + it("reads a .ts file as TypeScript, where a generic arrow is not JSX", () => { + const keys = functionsIn("a.ts", "const f = (a: T) => a;\nconst h = (a: T) => a;\n"); + expect(keys.map((span) => span.key)).toEqual(["a.ts::", "a.ts::f", "a.ts::h"]); + }); + + it("reads a .tsx file as TSX", () => { + const keys = functionsIn("a.tsx", "const v =
{[1].map((x) => x)}
;\n"); + expect(keys.map((span) => span.key)).toEqual(["a.tsx::", "a.tsx::#1"]); + }); + + it("ends the module where the text ends, with or without a last newline", () => { + expect(functionsIn("a.ts", "const a = 1;\nconst b = 2;\n")[0]?.end).toBe(2); + expect(functionsIn("a.ts", "const a = 1;\nconst b = 2;")[0]?.end).toBe(2); + expect(functionsIn("a.ts", "const a = 1;\n\n")[0]?.end).toBe(2); + }); + + it("picks the innermost span whatever order they come in, and the later of two of one size", () => { + const outer = { key: "o", start: 1, end: 10 }; + const inner = { key: "i", start: 3, end: 4 }; + expect(innermostAt([outer, inner], 3)?.key).toBe("i"); + expect(innermostAt([inner, outer], 3)?.key).toBe("i"); + expect(innermostAt([{ ...inner, key: "x" }, inner], 3)?.key).toBe("i"); + }); +}); + +describe("what git says when it fails", () => { + const failing = + (stderr: string, stdout = "", code = 1): Exec => + () => + Promise.resolve({ code, stdout, stderr }); + + it("is its own words, trimmed", async () => { + expect(await mergeBaseOf(failing(" boom \n"), "/x", "main")).toEqual({ + ok: false, + reason: "not a git work tree: boom", + }); + }); + + it("is the command that failed when git said nothing", async () => { + expect(await mergeBaseOf(failing(""), "/x", "main")).toEqual({ + ok: false, + reason: "not a git work tree: git rev-parse --is-inside-work-tree failed", + }); + }); + + it("is that there is no merge base when git printed none", async () => { + expect(await mergeBaseOf(failing("", "", 0), "/x", "main")).toEqual({ + ok: false, + reason: "there is no merge base between HEAD and main", + }); + }); +}); + +/** What the fake Vitest does with a run: fail a run of two or more files, else write a report that covers f. */ +function fakeVitest(args: readonly string[]): { code: number; stdout: string; stderr: string } { + const flag = args.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + const files = args.filter((arg) => arg.endsWith(".test.ts")); + if (files.length > 1) return { code: 1, stdout: "", stderr: "" }; + const loc = { start: { line: 1 }, end: { line: 3 } }; + const report = { "src/m.ts": { fnMap: { 0: { loc } }, f: { 0: 1 } } }; + write(flag.slice("--coverage.reportsDirectory=".length), { + "coverage-final.json": JSON.stringify(report), + }); + return { code: 0, stdout: "", stderr: "" }; +} + +/** Git is the given fake; Vitest is a fake that keeps its arguments. */ +function spied(git: Exec): { exec: Exec; vitest: string[][] } { + const vitest: string[][] = []; + const exec: Exec = (command, args, options) => { + if (command === "git") return git(command, args, options); + vitest.push([...args]); + return Promise.resolve(fakeVitest(args)); + }; + return { exec, vitest }; +} + +describe("the default runs of Vitest", () => { + const added = (): { cwd: string; git: Exec } => ({ + cwd: plainDir({ "src/m.ts": M, "src/a.test.ts": TEST, "src/b.test.ts": TEST }), + git: fakeGit( + diffOf([ + { path: "src/a.test.ts", kind: "added" }, + { path: "src/b.test.ts", kind: "added" }, + ]), + ), + }); + + it("runs both files together first, then each alone where that fails, under the gate's own options", async () => { + const project = added(); + const { exec, vitest } = spied(project.git); + const result = await diffScope({ cwd: project.cwd, base: "main", exec }); + expect(vitest.map((args) => args.filter((arg) => arg.endsWith(".test.ts")))).toEqual([ + ["src/a.test.ts", "src/b.test.ts"], + ["src/a.test.ts"], + ["src/b.test.ts"], + ]); + const first = vitest[0] ?? []; + expect(first.slice(0, 3)).toEqual(["run", "src/a.test.ts", "src/b.test.ts"]); + expect(first.filter((arg) => arg.startsWith("--coverage.exclude="))).toEqual( + DEFAULT_COVERAGE_EXCLUDES.map((glob) => `--coverage.exclude=${glob}`), + ); + expect(first).toContain(`--coverage.include=${COVERAGE_INCLUDE}`); + expect(first).toHaveLength(9 + DEFAULT_COVERAGE_EXCLUDES.length); + expect(result.outcome).toBe("scope"); + }); + + it("adds the exclusions the project gives, and removes its temporary directories", async () => { + const project = added(); + const { exec, vitest } = spied(project.git); + await diffScope({ cwd: project.cwd, base: "main", exec, exclude: ["gen/**"] }); + const first = vitest[0] ?? []; + expect(first).toContain("--coverage.exclude=gen/**"); + expect(first).toHaveLength(10 + DEFAULT_COVERAGE_EXCLUDES.length); + const flag = first.find((arg) => arg.startsWith("--coverage.reportsDirectory=")) ?? ""; + const out = flag.slice("--coverage.reportsDirectory=".length); + expect(basename(out).startsWith("wt-cov-")).toBe(true); + expect(existsSync(out)).toBe(false); + }); + + it("asks git for the merge-base version of a modified test, and of no added one", async () => { + const seen: string[][] = []; + const cwd = plainDir({ "src/m.ts": M, "src/a.test.ts": TEST, "src/b.test.ts": TEST }); + const diff = diffOf([ + { path: "src/a.test.ts", kind: "modified" }, + { path: "src/b.test.ts", kind: "added" }, + ]); + const { exec } = spied(fakeGit(diff, { "src/a.test.ts": TEST }, seen)); + await diffScope({ cwd, base: "main", exec }); + const shows = seen.filter((args) => args[0] === "show"); + expect(shows).toEqual([["show", `${FAKE_BASE}:src/a.test.ts`]]); + }); +}); + +describe("the merge-base copies", () => { + const modified = (path: string): { cwd: string; exec: Exec } => ({ + cwd: plainDir({ "src/m.ts": M, [path]: TEST }), + exec: fakeGit(diffOf([{ path, kind: "modified" }]), { [path]: TEST }), + }); + + it("are named before the extension for a test with no marker, after its last dot", async () => { + const asked: string[] = []; + await diffScope({ + ...modified("src/__tests__/a.b.ts"), + copyToken: () => "tok", + base: "main", + coverageOf: (file) => { + asked.push(file); + return Promise.resolve(ran(true, false)); + }, + }); + expect(asked).toEqual(["src/__tests__/a.b.ts", "src/__tests__/a.b.wt-base-tok.ts"]); + }); + + it("are removed even when the run took the copy away itself", async () => { + const project = modified("src/a.test.ts"); + const result = await diffScope({ + ...project, + copyToken: () => "tok", + base: "main", + coverageOf: (file) => { + if (file.includes(".wt-base-")) rmSync(join(project.cwd, file)); + return Promise.resolve(ran(true, false)); + }, + }); + expect(result.outcome).toBe("scope"); + }); +}); + +describe("the ranges to mutate", () => { + it("are not joined across two files, however close the last line of one and the first of the next", async () => { + const lines = (n: number, mark: string): string => + `${Array.from({ length: n }, (_, at) => `export const v${String(at)}${mark} = ${String(at)};`).join("\n")}\n`; + const cwd = plainDir({ "src/a.ts": lines(5, "a"), "src/b.ts": lines(8, "b") }); + const exec = fakeGit( + diffOf([ + { path: "src/a.ts", kind: "modified", range: [3, 5] }, + { path: "src/b.ts", kind: "modified", range: [6, 8] }, + ]), + ); + const result = await diffScope({ cwd, base: "main", exec }); + expect(result.outcome === "scope" ? result.scope.mutate : []).toEqual([ + "src/a.ts:3-5", + "src/b.ts:6-8", + ]); + }); +}); diff --git a/src/diff/real-batch.test.ts b/src/diff/real-batch.test.ts new file mode 100644 index 0000000..b1ea258 --- /dev/null +++ b/src/diff/real-batch.test.ts @@ -0,0 +1,126 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { REAL_TOOL_TIMEOUT } from "../gate/test-support-timeouts.ts"; +import { realProject } from "../layers/tests/testkit.ts"; +import { commit, git, write } from "./test-support.ts"; + +// With the real Vitest: the changed test files run together give the scope that a run of each alone gives, and the +// same verdict when one of them cannot run. Both scopes are taken in a child process, so that these runs are not +// part of what the mutation layer re-runs for every mutant of the diff step. + +const dirs: string[] = []; + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const KIT = join(import.meta.dirname, "../.."); + +const SCRIPT = ` +const [kit, cwd, token] = process.argv.slice(2); +const { diffScope } = await import(kit + "/src/diff/scope.ts"); +const { vitestCoverageOf } = await import(kit + "/src/diff/coverage.ts"); +const { realExec } = await import(kit + "/src/gate/exec.ts"); +const alone = (file) => vitestCoverageOf(file, { cwd, exec: realExec }); +const copyToken = () => token; +const before = await diffScope({ copyToken, cwd, base: "main", coverageOf: alone }); +const after = await diffScope({ copyToken, cwd, base: "main" }); +const scope = after.scope; +console.log(JSON.stringify({ before, after, keys: scope ? scope.coveredFunctions.map((ref) => ref.key) : [], mutate: scope ? scope.mutate : [] })); +`; + +const M = [ + "export function f(a: number): number {", + " return a + 1;", + "}", + "export function g(a: number): number {", + " return a * 2;", + "}", + "export function h(a: number): number {", + " return a - 1;", + "}", + "", +].join("\n"); +const test = (calls: string): string => + `import { expect, it } from "vitest";\nimport { f, g, h } from "./m.ts";\nit("runs", () => {\n expect(${calls}).toBeTypeOf("number");\n});\n`; +const BREAKS = + 'import { it } from "vitest";\nit("breaks", () => {\n throw new Error("no");\n});\n'; + +function branch(files: Readonly>): string { + const { project } = realProject({ ".gitignore": "node_modules\n.web-tops\n", ...files }); + dirs.push(project.dir); + git(project.dir, "init", "-q", "-b", "main"); + commit(project.dir, "base"); + git(project.dir, "switch", "-q", "-c", "work"); + return project.dir; +} + +interface Scopes { + readonly before: unknown; + readonly after: unknown; + readonly keys: readonly unknown[]; + readonly mutate: readonly unknown[]; +} + +const listOf = (value: unknown): unknown[] => (Array.isArray(value) ? value : []); + +/** Both scopes of the project, as the child process took them: each file alone, and the files together. */ +function scopes(cwd: string): Scopes { + const dir = mkdtempSync(join(tmpdir(), "wt-real-batch-")); + dirs.push(dir); + const script = join(dir, "scopes.mjs"); + writeFileSync(script, SCRIPT); + const run = spawnSync("bun", [script, KIT, cwd, "tok"], { encoding: "utf8", cwd }); + expect(run.status, run.stderr).toBe(0); + const parsed: unknown = JSON.parse(run.stdout); + const field = (name: string): unknown => Reflect.get(Object(parsed), name); + return { + before: field("before"), + after: field("after"), + keys: listOf(field("keys")), + mutate: listOf(field("mutate")), + }; +} + +describe("the real run of changed test files together", () => { + it( + "is the scope of a run of each alone: added, weakened and unchanged-source tests", + () => { + const cwd = branch({ "src/m.ts": M, "src/weak.test.ts": test("f(1) + g(1)") }); + write(cwd, { + "src/weak.test.ts": test("f(1)"), + "src/a.test.ts": test("g(2)"), + "src/b.test.ts": test("f(3)"), + }); + commit(cwd, "work"); + const { before, after, keys, mutate } = scopes(cwd); + expect(after).toEqual(before); + expect(keys).toEqual(["src/m.ts::f", "src/m.ts::g"]); + expect(mutate).toEqual(["src/m.ts:1-6"]); + }, + REAL_TOOL_TIMEOUT * 3, + ); + + it( + "fails as a run of each alone fails, naming the first file that cannot run", + () => { + const cwd = branch({ "src/m.ts": M }); + write(cwd, { + "src/a.test.ts": test("f(1)"), + "src/b.test.ts": BREAKS, + "src/c.test.ts": BREAKS, + }); + commit(cwd, "work"); + const { before, after } = scopes(cwd); + expect(after).toEqual(before); + expect(after).toEqual({ + outcome: "fail", + reason: "could not run src/b.test.ts to find the functions it covers", + }); + }, + REAL_TOOL_TIMEOUT * 3, + ); +}); diff --git a/src/diff/scope.ts b/src/diff/scope.ts index ec4f4f8..9b7a341 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -3,7 +3,12 @@ import { readFileSync, rmSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import { realExec } from "../gate/exec.ts"; import type { Exec } from "../gate/types.ts"; -import { vitestCoverageOf, coveredSpans } from "./coverage.ts"; +import { + vitestCoverageOf, + vitestCoverageOfMany, + coveredSpans, + type CoverageRun, +} from "./coverage.ts"; import { functionsIn, innermostAt } from "./functions.ts"; import { diffText, mergeBaseOf, showAt } from "./git.ts"; import { parseDiff } from "./parse.ts"; @@ -18,6 +23,9 @@ import type { export type CoverageOf = (testFile: string) => Promise; +/** What several test files cover together: one run for all of them, the functions any of them ran. */ +export type CoverageOfMany = (testFiles: readonly string[]) => Promise; + export interface ScopeOptions { readonly cwd: string; /** The ref the run is judged against, such as `origin/main`. */ @@ -25,6 +33,11 @@ export interface ScopeOptions { readonly exec?: Exec; /** What a test file covers; by default a Vitest run of that file alone. */ readonly coverageOf?: CoverageOf; + /** + * What test files cover together, in one run: used, when given, before `coverageOf`, which then only runs the files of + * a group that did not run together. By default a Vitest run of a whole group, when `coverageOf` is not given either. + */ + readonly coverageOfMany?: CoverageOfMany; /** Reasoned coverage exclusions from `web-tops.yaml`, passed to the default per-file run. */ readonly exclude?: readonly string[]; /** The random part of a merge-base copy's name; a test sets it to know the name. */ @@ -43,13 +56,13 @@ type Covered = type Reader = (path: string) => string | undefined; -const TEST = /\.(test|spec)\.[cm]?tsx?$|(^|\/)__tests__\//; -const SOURCE = /\.[cm]?tsx?$/; -const DECLARATION = /\.d\.[cm]?ts$/; +const hasScriptExtension = (path: string): boolean => /\.[cm]?tsx?$/.test(path); +const isDeclaration = (path: string): boolean => /\.d\.[cm]?ts$/.test(path); -export const isTestFile = (path: string): boolean => TEST.test(path) && SOURCE.test(path); +export const isTestFile = (path: string): boolean => + /\.(test|spec)\.[cm]?tsx?$|(^|\/)__tests__\//.test(path) && hasScriptExtension(path); export const isSourceFile = (path: string): boolean => - SOURCE.test(path) && !DECLARATION.test(path) && !TEST.test(path); + hasScriptExtension(path) && !isDeclaration(path) && !isTestFile(path); function readerFor(cwd: string): Reader { return (path) => { @@ -101,38 +114,88 @@ function refsCovered(map: CoverageMap, read: Reader): readonly FunctionRef[] { }); } -interface CoverRun { - readonly coverageOf: CoverageOf; - readonly read: Reader; - /** The coverage of a test file as it was at the merge base, or undefined when it has none or cannot run. */ - readonly baseCoverageOf: CoverageOf; +/** How many test files one run covers at most, so that a very large diff never overruns the length of a command line. */ +const GROUP = 100; + +function groupsOf(paths: readonly string[]): string[][] { + return Array.from({ length: Math.ceil(paths.length / GROUP) }, (_, at) => + paths.slice(at * GROUP, (at + 1) * GROUP), + ); } -const NONE: CoverageMap = new Map(); +interface Gathered { + readonly maps: readonly CoverageMap[]; + /** The files that could not run alone, in the order given. */ + readonly failed: readonly string[]; +} + +/** What the files cover, one run each: a file that cannot run is `failed`. */ +async function alone(group: readonly string[], one: CoverageOf): Promise { + const maps: CoverageMap[] = []; + const failed: string[] = []; + for (const path of group) { + const got = await one(path); + if (got === undefined) failed.push(path); + else maps.push(got); + } + return { maps, failed }; +} + +/** A group in one run when `many` is given, and file by file where that run gives nothing. */ +async function groupCoverage( + group: readonly string[], + one: CoverageOf, + many: CoverageOfMany | undefined, +): Promise { + const together = many === undefined ? undefined : await many(group); + return together === undefined ? alone(group, one) : { maps: [together], failed: [] }; +} + +/** + * What `paths` cover. Each group of them runs together when `many` is given; a group that does not run together (one + * of its files fails, or the run leaves nothing) runs file by file, as every file did before, so a file that cannot + * run is found and named whatever the others do. + */ +async function gather( + paths: readonly string[], + one: CoverageOf, + many: CoverageOfMany | undefined, +): Promise { + const found: Gathered[] = []; + for (const group of groupsOf(paths)) found.push(await groupCoverage(group, one, many)); + return { maps: found.flatMap((got) => got.maps), failed: found.flatMap((got) => got.failed) }; +} -async function headMap(test: ChangedFile, run: CoverRun): Promise { - return test.status === "deleted" ? NONE : run.coverageOf(test.path); +interface Sources { + readonly coverageOf: CoverageOf; + readonly coverageOfMany: CoverageOfMany | undefined; } -async function baseMap(test: ChangedFile, run: CoverRun): Promise { - return test.status === "added" ? NONE : ((await run.baseCoverageOf(test.path)) ?? NONE); +interface CoverRun extends Sources { + readonly read: Reader; + /** The merge-base versions of the changed test files, run beside the originals, as they covered then. */ + readonly baseMaps: (tests: readonly ChangedFile[]) => Promise; } +const byPathThenStartRef = (a: FunctionRef, b: FunctionRef): number => + a.path.localeCompare(b.path) || a.start - b.start; + /** * The functions every changed test file covers, as it is at HEAD and as it was at the merge base (so a test - * that dropped a call keeps the function it no longer calls in scope). A head version that cannot run is `fail`; - * a merge-base version that cannot run against the head is left out. + * that dropped a call keeps the function it no longer calls in scope). A head version that cannot run is `fail`, + * naming the first such file; a merge-base version that cannot run against the head is left out. The functions are + * those any of the runs covered, ordered by file and line. */ async function coveredFunctions(tests: readonly ChangedFile[], run: CoverRun): Promise { - const refs: FunctionRef[] = []; - for (const test of tests) { - const head = await headMap(test, run); - if (head === undefined) { - return { ok: false, reason: `could not run ${test.path} to find the functions it covers` }; - } - refs.push(...refsCovered(head, run.read), ...refsCovered(await baseMap(test, run), run.read)); + const heads = tests.filter((test) => test.status !== "deleted").map((test) => test.path); + const head = await gather(heads, run.coverageOf, run.coverageOfMany); + const [first] = head.failed; + if (first !== undefined) { + return { ok: false, reason: `could not run ${first} to find the functions it covers` }; } - return { ok: true, refs: uniqueByKey(refs) }; + const bases = await run.baseMaps(tests.filter((test) => test.status !== "added")); + const refs = [...head.maps, ...bases].flatMap((map) => refsCovered(map, run.read)); + return { ok: true, refs: [...uniqueByKey(refs)].sort(byPathThenStartRef) }; } const randomToken = (): string => randomBytes(6).toString("hex"); @@ -161,36 +224,80 @@ function createCopy(path: string, text: string): boolean { } } -/** - * The coverage of the merge-base copy of a test file, written beside the original so its imports resolve, run, - * and removed. Only a file this created is removed; a path that is already taken means no run (undefined). - */ -async function baseCoverage( +interface Copy { + /** The copy's path relative to the project, as the test runner is asked for it. */ + readonly name: string; + readonly path: string; +} + +/** The merge-base copy of one test file, written beside it so its imports resolve; undefined when the file is not at the merge base or the name is taken. */ +async function copyOf( options: ScopeOptions, exec: Exec, mergeBase: string, - coverageOf: CoverageOf, test: string, -): Promise { +): Promise { const shown = await showAt(exec, options.cwd, mergeBase, test); if (!shown.ok) return undefined; - const copy = copyName(test, (options.copyToken ?? randomToken)()); - const path = join(options.cwd, copy); - return createCopy(path, shown.value) ? runCopy(coverageOf, copy, path) : undefined; + const name = copyName(test, (options.copyToken ?? randomToken)()); + const path = join(options.cwd, name); + return createCopy(path, shown.value) ? { name, path } : undefined; } -/** Runs the coverage of a copy this created, and removes the copy whatever happens. */ -async function runCopy( - coverageOf: CoverageOf, - copy: string, - path: string, +/** What a coverage source gives, or undefined for a run that throws. */ +async function quietly( + run: (given: A) => Promise, + given: A, ): Promise { try { - return await coverageOf(copy); + return await run(given); } catch { return undefined; + } +} + +function quietMany(many: CoverageOfMany | undefined): CoverageOfMany | undefined { + return many === undefined ? undefined : (group) => quietly(many, group); +} + +/** Writes the merge-base copy of each test file that has one, adding each to `copies` as it is made. */ +async function copyAll( + copies: Copy[], + options: ScopeOptions, + exec: Exec, + mergeBase: string, + tests: readonly ChangedFile[], +): Promise { + for (const test of tests) { + const copy = await copyOf(options, exec, mergeBase, test.path); + if (copy !== undefined) copies.push(copy); + } +} + +/** + * The coverage of the merge-base versions of the changed test files, each written beside the original under a name + * that is never the original's, run together (file by file where that fails), and all removed whatever happens. Only a + * file this created is removed; a path that is already taken means no run for that file. + */ +async function baseCoverage( + options: ScopeOptions, + exec: Exec, + mergeBase: string, + sources: Sources, + tests: readonly ChangedFile[], +): Promise { + const copies: Copy[] = []; + try { + await copyAll(copies, options, exec, mergeBase, tests); + const names = copies.map((copy) => copy.name); + const got = await gather( + names, + (file) => quietly(sources.coverageOf, file), + quietMany(sources.coverageOfMany), + ); + return got.maps; } finally { - rmSync(path, { force: true }); + for (const copy of copies) rmSync(copy.path, { force: true }); } } @@ -239,6 +346,21 @@ function scopeOf( }; } +/** A source for a group of files: the caller's, else Vitest on the group, unless the caller gave a source for one file only. */ +function manyOf(options: ScopeOptions, run: CoverageRun): CoverageOfMany | undefined { + if (options.coverageOfMany !== undefined) return options.coverageOfMany; + return options.coverageOf === undefined ? (group) => vitestCoverageOfMany(group, run) : undefined; +} + +/** Where coverage comes from: what the caller gave, else Vitest runs. */ +function sourcesOf(options: ScopeOptions, exec: Exec): Sources { + const run = { cwd: options.cwd, exec, exclude: options.exclude ?? [] }; + return { + coverageOf: options.coverageOf ?? ((test) => vitestCoverageOf(test, run)), + coverageOfMany: manyOf(options, run), + }; +} + async function assemble( options: ScopeOptions, exec: Exec, @@ -246,14 +368,13 @@ async function assemble( files: readonly ChangedFile[], ): Promise { const read = readerFor(options.cwd); - const coverageOf = - options.coverageOf ?? - ((test: string) => - vitestCoverageOf(test, { cwd: options.cwd, exec, exclude: options.exclude ?? [] })); - const baseCoverageOf = (test: string): Promise => - baseCoverage(options, exec, mergeBase, coverageOf, test); + const coverage = sourcesOf(options, exec); const tests = files.filter((file) => isTestFile(file.path)); - const covered = await coveredFunctions(tests, { coverageOf, read, baseCoverageOf }); + const covered = await coveredFunctions(tests, { + ...coverage, + read, + baseMaps: (changed) => baseCoverage(options, exec, mergeBase, coverage, changed), + }); if (!covered.ok) return { outcome: "fail", reason: covered.reason }; const scope = scopeOf(mergeBase, files, read, covered.refs); return scope.mutate.length === 0 diff --git a/src/diff/test-support.ts b/src/diff/test-support.ts index 7478c18..44db909 100644 --- a/src/diff/test-support.ts +++ b/src/diff/test-support.ts @@ -1,3 +1,4 @@ +import type { Exec, ExecResult } from "../gate/types.ts"; import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -60,6 +61,83 @@ export function commit(dir: string, message: string): void { git(dir, "commit", "-q", "-m", message); } +/** A directory with the given files and no repository: for a run whose git is a fake. */ +export function plainDir(files: Readonly>): string { + const dir = mkdtempSync(join(tmpdir(), "wt-diff-")); + made.push(dir); + write(dir, files); + return dir; +} + +export interface Edit { + readonly path: string; + readonly kind: "added" | "modified"; + /** The new-side lines the edit touches; line 1 by default. */ + readonly range?: readonly [number, number]; +} + +/** A `git diff -U0` text for the edits: what the diff step would read from git. */ +export function diffOf(edits: readonly Edit[]): string { + const blocks = edits.map((edit) => { + const [start, end] = edit.range ?? [1, 1]; + const count = end - start + 1; + const old = edit.kind === "added" ? "/dev/null" : `a/${edit.path}`; + const hunk = + edit.kind === "added" + ? `@@ -0,0 +${String(start)},${String(count)} @@` + : `@@ -${String(start)},0 +${String(start)},${String(count)} @@`; + const added = Array.from({ length: count }, () => "+x"); + return [ + `diff --git a/${edit.path} b/${edit.path}`, + `--- ${old}`, + `+++ b/${edit.path}`, + hunk, + ...added, + ].join("\n"); + }); + return `${blocks.join("\n")}\n`; +} + +/** The merge base the fake git gives. */ +export const FAKE_BASE = "abc1234"; + +/** + * A git that says the project is a full checkout on a branch whose merge base is `FAKE_BASE`, and gives `diff` as the + * diff and `atBase` as the files at the merge base. It keeps the arguments of every call in `seen`. + */ +export function fakeGit( + diff: string, + atBase: Readonly> = {}, + seen: string[][] = [], +): Exec { + return (_command, args) => { + seen.push([...args]); + return Promise.resolve(answerOf(args, diff, atBase)); + }; +} + +const said = (stdout: string): ExecResult => ({ code: 0, stdout, stderr: "" }); + +const ANSWERS: Readonly> = { + "--is-shallow-repository": "false", + "--is-inside-work-tree": "true", +}; + +function showOf(args: readonly string[], atBase: Readonly>): ExecResult { + const text = atBase[args[1]?.slice(FAKE_BASE.length + 1) ?? ""]; + return text === undefined ? { code: 128, stdout: "", stderr: "no such path" } : said(text); +} + +function answerOf( + args: readonly string[], + diff: string, + atBase: Readonly>, +): ExecResult { + if (args[0] === "show") return showOf(args, atBase); + if (args.includes("diff")) return said(diff); + return said(args.map((arg) => ANSWERS[arg]).find((answer) => answer !== undefined) ?? FAKE_BASE); +} + export function cleanRepos(): void { for (const dir of made.splice(0)) rmSync(dir, { recursive: true, force: true }); }