diff --git a/src/gate/handoffs.test.ts b/src/gate/handoffs.test.ts index 23312ca..9bb5678 100644 --- a/src/gate/handoffs.test.ts +++ b/src/gate/handoffs.test.ts @@ -5,7 +5,7 @@ import type { Report } from "../report/types.ts"; import { runGate } from "./runner.ts"; import { buildRegistry } from "./registry.ts"; import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; -import type { GateConfig, GateOptions, Layer } from "./types.ts"; +import type { Consumed, GateConfig, GateOptions, Layer } from "./types.ts"; /** R29 N5: layers declare the artifacts they produce and consume; a missing input is `not-run`, a present one is judged. */ @@ -307,12 +307,19 @@ describe("G2c the real registry declares its hand-offs", () => { const layers = buildRegistry(cfg()); const byId = new Map(layers.map((layer) => [layer.id, layer])); - it("never consumes from a layer it does not require, and the producer declares what is consumed", () => { - for (const layer of layers) { - for (const need of layer.consumes) { - expect(layer.requires).toContain(need.from); - expect(byId.get(need.from)?.produces).toContain(need.artifact); - } + const isRequired = (need: Consumed): boolean => need.optional !== true; + + it("never consumes from a layer it does not require", () => { + const needs = layers.flatMap((layer) => layer.consumes.map((need) => [layer, need] as const)); + for (const [layer, need] of needs) { + expect(layer.requires).toContain(need.from); + } + }); + + it("the producer declares what a layer consumes, but an optional input may be one it does not write (R92)", () => { + const needs = layers.flatMap((layer) => layer.consumes.filter(isRequired)); + for (const need of needs) { + expect(byId.get(need.from)?.produces).toContain(need.artifact); } }); diff --git a/src/gate/plan-edges.test.ts b/src/gate/plan-edges.test.ts new file mode 100644 index 0000000..1f0e294 --- /dev/null +++ b/src/gate/plan-edges.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { planGate } from "./plan.ts"; +import { cfg, fakeLayer, opts } from "./testkit.ts"; +import type { GateConfig } from "./types.ts"; + +/** The plan's edges: a switch with no reason, layers that require each other, and ids nothing registers. */ + +const enabled = (id: string): GateConfig => cfg({ toggles: { [id]: { enabled: true } } }); + +describe("the reason a layer is skipped", () => { + it("is the config's own words, and says so when the config gave none", () => { + const layers = [fakeLayer({ id: "hygiene" })]; + const bare = cfg({ toggles: { hygiene: { enabled: false } } }); + expect(planGate(layers, bare, opts())[0]?.skip).toBe("disabled in web-tops.yaml"); + const said = cfg({ toggles: { hygiene: { enabled: false, reason: "lands later" } } }); + expect(planGate(layers, said, opts())[0]?.skip).toBe("lands later"); + }); +}); + +describe("--only follows what a layer requires", () => { + it("ends when two layers require each other", () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, requires: ["tests"] }), + fakeLayer({ id: "tests", gate: 2, requires: ["hygiene"] }), + fakeLayer({ id: "coverage", gate: 3 }), + ]; + const plan = planGate(layers, cfg(), opts({ only: "hygiene" })); + expect(plan.map((item) => [item.id, item.skip, item.requiredBy])).toEqual([ + ["hygiene", undefined, undefined], + ["tests", undefined, "hygiene"], + ["coverage", "not selected by --only hygiene", undefined], + ]); + }); + + it("pulls a layer in once, by the first that needs it", () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1 }), + fakeLayer({ id: "tests", gate: 2, requires: ["hygiene"] }), + fakeLayer({ id: "coverage", gate: 3, requires: ["tests", "hygiene"] }), + ]; + const plan = planGate(layers, cfg(), opts({ only: "coverage" })); + expect(plan.map((item) => item.requiredBy)).toEqual(["coverage", "coverage", undefined]); + }); + + it("names a layer nothing registers and pulls nothing else in", () => { + const layers = [fakeLayer({ id: "hygiene", gate: 1 })]; + const plan = planGate(layers, cfg(), opts({ only: "ghost" })); + expect(plan.map((item) => [item.id, item.skip])).toEqual([ + ["hygiene", "not selected by --only ghost"], + ]); + }); +}); + +describe("an id the config enables and nothing registers", () => { + const layers = [fakeLayer({ id: "hygiene", gate: 1 })]; + + it("is in the plan as a layer of its own, and not when --only picks a layer", () => { + expect(planGate(layers, enabled("ghost"), opts()).map((item) => item.id)).toEqual([ + "hygiene", + "ghost", + ]); + expect( + planGate(layers, enabled("ghost"), opts({ only: "hygiene" })).map((item) => item.id), + ).toEqual(["hygiene"]); + }); + + it("is skipped for the reason any layer would be, such as a tier above the one asked", () => { + const plan = planGate(layers, enabled("ghost"), opts({ maxTier: 0 })); + expect(plan.find((item) => item.id === "ghost")?.skip).toBe("tier 1 is above --max-tier 0"); + }); +}); diff --git a/src/gate/plan.ts b/src/gate/plan.ts index 7e80d02..0b6799a 100644 --- a/src/gate/plan.ts +++ b/src/gate/plan.ts @@ -33,7 +33,7 @@ interface Scope { } function requiresOf(layers: readonly Layer[], id: string): readonly string[] { - return layers.find((layer) => layer.id === id)?.requires ?? []; + return layers.filter((layer) => layer.id === id).flatMap((layer) => layer.requires); } function select( diff --git a/src/gate/ratchet-mode.test.ts b/src/gate/ratchet-mode.test.ts new file mode 100644 index 0000000..5dbd2ec --- /dev/null +++ b/src/gate/ratchet-mode.test.ts @@ -0,0 +1,61 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Baseline } from "../baseline/types.ts"; +import { runGate } from "./runner.ts"; +import { cfg, fakeLayer, noExec, opts, tickingClock } from "./testkit.ts"; + +/** Only the ratchet mode reads the baseline a config names. */ + +const made: string[] = []; + +afterEach(() => { + for (const dir of made.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const BASELINE = ".web-tops/baseline.json"; + +const BASE: Baseline = { + report_version: "2.0.0", + subject: { package: "fixture", root: "." }, + crap: { "src/a.ts::f": { complexity: 3, coverage: 1, crap: 3 } }, + coverage: {}, + findings: {}, + fingerprints: [], + census: {}, + pages: {}, +}; + +function above(enforce: "hard"): ReturnType { + const dir = mkdtempSync(join(tmpdir(), "wt-ratchet-mode-")); + made.push(dir); + mkdirSync(join(dir, ".web-tops")); + writeFileSync(join(dir, BASELINE), JSON.stringify(BASE)); + const crap = fakeLayer({ + id: "crap", + gate: 4, + metrics: { + crap: [{ file: "src/a.ts", name: "f", line: 1, complexity: 3, coverage: 1, crap: 9 }], + }, + }); + return runGate({ + layers: [crap], + config: cfg({ enforce, baselinePath: BASELINE }), + options: opts(), + cwd: dir, + env: {}, + exec: noExec, + outDir: join(dir, ".web-tops", "out"), + now: tickingClock(), + }); +} + +describe("G9b: a run above its baseline", () => { + it("is left alone under hard, though the config names a baseline", async () => { + const hard = await above("hard"); + expect(hard.layers[0]?.verdict).toBe("pass"); + expect(hard.layers[0]?.findings).toEqual([]); + expect(hard.passed).toBe(true); + }); +}); diff --git a/src/gate/registry-order.test.ts b/src/gate/registry-order.test.ts new file mode 100644 index 0000000..eb6e81e --- /dev/null +++ b/src/gate/registry-order.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { buildRegistry } from "./registry.ts"; +import { compareGates } from "./order.ts"; +import { customLayer } from "./stubs.ts"; +import { cfg, noExec } from "./testkit.ts"; +import type { LayerContext } from "./types.ts"; + +describe("the registry lists the layers in gate order", () => { + it("with the config's custom layers between gate 6 and laws", () => { + const config = cfg({ + custom: [ + { id: "extra", tier: 1, command: "echo" }, + { id: "more", tier: 2 }, + ], + }); + const layers = buildRegistry(config); + expect(layers.map((layer) => layer.id)).toEqual([ + "hygiene", + "tests", + "coverage", + "crap", + "mutation", + "build-output", + "extra", + "more", + "laws", + ]); + const gates = layers.map((layer) => layer.gate); + expect(gates).toEqual([...gates].sort(compareGates)); + }); +}); + +describe("a custom layer is a gate 6 layer that is not run yet", () => { + it("needs and gives nothing, for every launch class, and is not-run with the reason it has no command", async () => { + const mapped = customLayer("extra", 2, "echo"); + expect(mapped).toMatchObject({ + id: "extra", + gate: 6, + tier: 2, + requires: [], + produces: [], + consumes: [], + }); + expect(mapped.classes).toEqual([ + "ts-library", + "ui-components", + "astro-site", + "web-app", + "node-service", + ]); + const ctx: LayerContext = { + cwd: ".", + config: cfg(), + base: "origin/main", + enforce: "hard", + env: {}, + exec: noExec, + outDir: ".", + artifacts: new Map(), + }; + const results = [await mapped.run(ctx), await customLayer("bare", 1, undefined).run(ctx)]; + expect(results.map((result) => result.verdict)).toEqual(["not-run", "not-run"]); + expect(results.map((result) => result.reason)).toEqual([ + "custom layers run from M4", + "no command is mapped to this layer", + ]); + expect(results.map((result) => result.findings)).toEqual([[], []]); + }); +}); diff --git a/src/gate/registry.ts b/src/gate/registry.ts index 90eeed0..4740f41 100644 --- a/src/gate/registry.ts +++ b/src/gate/registry.ts @@ -1,10 +1,11 @@ import { coverageLayer } from "../layers/coverage/index.ts"; import { crapLayer } from "../layers/crap/index.ts"; +import { yamlCatalog } from "../laws/layer/catalog.ts"; +import { lawsLayer } from "../laws/layer/index.ts"; import { mutationLayer } from "../layers/mutation/index.ts"; import { hygieneLayer } from "../layers/hygiene/index.ts"; import { testsLayer } from "../layers/tests/index.ts"; -import { compareGates } from "./order.ts"; -import { BUILD_OUTPUT_STUB, customLayer, LAWS_STUB } from "./stubs.ts"; +import { BUILD_OUTPUT_STUB, customLayer } from "./stubs.ts"; import type { GateConfig, Layer } from "./types.ts"; /** The layers of one run: the seven built-ins and the config's custom layers, in gate order. */ @@ -18,6 +19,6 @@ export function buildRegistry(config: GateConfig): Layer[] { mutationLayer, BUILD_OUTPUT_STUB, ...custom, - LAWS_STUB, - ].sort((a, b) => compareGates(a.gate, b.gate)); + lawsLayer(yamlCatalog), + ]; } diff --git a/src/gate/runner-edges.test.ts b/src/gate/runner-edges.test.ts new file mode 100644 index 0000000..33a57dc --- /dev/null +++ b/src/gate/runner-edges.test.ts @@ -0,0 +1,247 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Report } from "../report/types.ts"; +import { optionsFor, runGate, sourceReader } from "./runner.ts"; +import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; +import type { GateConfig, GateOptions, Layer, LayerContext } from "./types.ts"; + +/** The runner's edges: what a layer entry holds when nothing ran, when a layer threw, and what each input and stop does. */ + +const projects: ReturnType[] = []; + +afterEach(() => { + for (const project of projects.splice(0)) { + project.cleanup(); + } +}); + +function setup(): { dir: string; write: (name: string) => string; run: Run } { + const project = tempProject(); + projects.push(project); + const outDir = join(project.dir, ".web-tops", "out"); + return { + dir: project.dir, + write: (name) => { + const path = join(outDir, name); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "{}\n"); + return path; + }, + run: (layers, config = cfg(), options = opts()) => + runGate({ + layers, + config, + options, + cwd: project.dir, + env: {}, + exec: noExec, + outDir, + now: tickingClock(), + }), + }; +} + +type Run = ( + layers: readonly Layer[], + config?: GateConfig, + options?: GateOptions, +) => Promise; + +function entry(report: Report, id: string): Report["layers"][number] { + const found = report.layers.find((layer) => layer.id === id); + if (found === undefined) { + throw new Error(`no ${id} in the report`); + } + return found; +} + +describe("the entry of a layer that did not run", () => { + it("has no findings, metrics or artifacts and took no time", async () => { + const config = cfg({ toggles: { hygiene: off("not for this project") } }); + const report = await setup().run([fakeLayer({ id: "hygiene" })], config); + const hygiene = entry(report, "hygiene"); + expect(hygiene.verdict).toBe("skip"); + expect(hygiene.findings).toStrictEqual([]); + expect(hygiene.metrics).toStrictEqual({}); + expect(hygiene.artifacts).toStrictEqual([]); + expect(hygiene.duration_ms).toBe(0); + }); +}); + +describe("a layer that throws", () => { + it("fails with what it threw, as a finding of its own, in the time it took", async () => { + const report = await setup().run([fakeLayer({ id: "hygiene", throws: new Error("boom") })]); + const hygiene = entry(report, "hygiene"); + expect(hygiene.verdict).toBe("fail"); + expect(hygiene.reason).toBe("layer threw: boom"); + expect(hygiene.findings).toHaveLength(1); + expect(hygiene.findings[0]).toMatchObject({ + tool: "webtops", + rule: "layer-threw", + verdict: "fail", + message: "boom", + }); + expect(hygiene.duration_ms).toBeGreaterThan(0); + expect(hygiene.duration_ms).toBeLessThan(100); + }); + + it("names what is thrown when it is not an Error", async () => { + const thrower: Layer = { + ...fakeLayer({ id: "hygiene" }), + run: () => Promise.reject(new Error("plain")), + }; + const report = await setup().run([thrower]); + expect(entry(report, "hygiene").reason).toBe("layer threw: plain"); + }); +}); + +describe("the time a layer took", () => { + it("is what the clock advanced while it ran", async () => { + const report = await setup().run([fakeLayer({ id: "hygiene" })]); + const took = entry(report, "hygiene").duration_ms; + expect(took).toBeGreaterThan(0); + expect(took).toBeLessThan(100); + }); +}); + +describe("a missing input names the layer that should have produced it", () => { + it("is the producer's own verdict, not the first layer's", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1 }), + fakeLayer({ id: "tests", gate: 2, verdict: "fail", reason: "a test failed" }), + fakeLayer({ + id: "coverage", + gate: 3, + requires: ["tests"], + consumes: [{ artifact: "coverage-json", from: "tests" }], + }), + ]; + const report = await setup().run(layers, cfg(), opts({ keepGoing: true })); + expect(entry(report, "coverage").reason).toBe("input coverage-json missing: tests fail"); + }); +}); + +/** A layer that records the inputs the runner showed it. */ +function watching(seen: string[][], consumes: Layer["consumes"]): Layer { + return { + ...fakeLayer({ id: "coverage", gate: 3, requires: ["hygiene"], consumes }), + run: (ctx: LayerContext) => { + seen.push([...ctx.artifacts.keys()]); + return Promise.resolve({ verdict: "pass", findings: [], artifacts: {} }); + }, + }; +} + +describe("R92 an optional input", () => { + const optional = [{ artifact: "audit-json", from: "hygiene", optional: true }]; + + it("is passed when the producer left it, and the layer runs without it when not", async () => { + const seen: string[][] = []; + const home = setup(); + const left = fakeLayer({ + id: "hygiene", + artifacts: { "audit-json": home.write("hygiene/audit-json.json") }, + }); + const withIt = await home.run([left, watching(seen, optional)]); + expect(entry(withIt, "coverage").verdict).toBe("pass"); + const without = await setup().run([fakeLayer({ id: "hygiene" }), watching(seen, optional)]); + expect(entry(without, "coverage").verdict).toBe("pass"); + expect(seen).toEqual([["audit-json"], []]); + }); + + it("is not a missing input, while a required one beside it still is", async () => { + const seen: string[][] = []; + const both = [...optional, { artifact: "tsc-diagnostics", from: "hygiene" }]; + const report = await setup().run([fakeLayer({ id: "hygiene" }), watching(seen, both)]); + expect(entry(report, "coverage")).toMatchObject({ + verdict: "not-run", + reason: "input tsc-diagnostics missing: hygiene pass", + }); + expect(seen).toEqual([]); + }); +}); + +describe("after a stop, the laws layer judges only when every layer it requires has run", () => { + it("is not run when one of them was stopped before it ran", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "coverage", gate: 3 }), + fakeLayer({ id: "laws", gate: "laws", requires: ["hygiene", "coverage"] }), + ]; + const report = await setup().run(layers); + expect(entry(report, "laws")).toMatchObject({ + verdict: "not-run", + reason: "stopped: gate 1 failed", + }); + }); + + it("judges when the one it requires has run, whatever its verdict", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "coverage", gate: 3 }), + fakeLayer({ id: "laws", gate: "laws", requires: ["hygiene"] }), + ]; + const report = await setup().run(layers); + expect(entry(report, "laws").verdict).toBe("pass"); + expect(entry(report, "coverage").verdict).toBe("not-run"); + }); +}); + +describe("a run that stopped on a failing gate", () => { + it("fails with no problem listed: the layers it never reached are named by their own entries", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "tests", gate: 2 }), + ]; + const report = await setup().run(layers); + expect(report.problems).toEqual([]); + expect(report.passed).toBe(false); + expect(entry(report, "tests").verdict).toBe("not-run"); + }); +}); + +describe("the CRAP scope the report names", () => { + const all = cfg({ crap: { cap: 4, scope: "all", nullishDiscount: true } }); + + it("is diff by default, and all from the config or --all", async () => { + const layers = [fakeLayer({ id: "hygiene" })]; + expect((await setup().run(layers)).crap_scope).toBe("diff"); + expect((await setup().run(layers, all)).crap_scope).toBe("all"); + expect((await setup().run(layers, cfg(), opts({ all: true }))).crap_scope).toBe("all"); + }); +}); + +describe("the text of a file a finding points at", () => { + it("is read once however many findings point into it, and by file", () => { + const reads: string[] = []; + const read = sourceReader("/work", (cwd, file) => { + reads.push(`${cwd}:${file}`); + return file === "a.ts" ? "text" : undefined; + }); + expect([read("a.ts"), read("a.ts"), read("b.ts"), read("b.ts"), read("a.ts")]).toEqual([ + "text", + "text", + undefined, + undefined, + "text", + ]); + expect(reads).toEqual(["/work:a.ts", "/work:b.ts"]); + }); +}); + +describe("the diff engine's options for a layer", () => { + const base = { cwd: "/work", base: "main", exclude: ["src/gen/**"] }; + + it("are kept for a layer that needs the tests' functions", () => { + expect(optionsFor({ coveredByTests: true }, base)).toBe(base); + }); + + it("add a coverage of none for one that does not, and keep the rest", async () => { + const given = optionsFor({ coveredByTests: false }, base); + expect(given).toMatchObject(base); + const covered = await given.coverageOf?.("src/a.test.ts"); + expect(covered).toBeInstanceOf(Map); + expect(covered?.size).toBe(0); + }); +}); diff --git a/src/gate/runner.ts b/src/gate/runner.ts index ddf7a95..eab775b 100644 --- a/src/gate/runner.ts +++ b/src/gate/runner.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join, relative, resolve, sep } from "node:path"; import { applyRatchet } from "../baseline/ratchet.ts"; -import { diffScope, type CoverageMap, type ScopeResult } from "../diff/index.ts"; +import { diffScope, type CoverageMap, type ScopeOptions, type ScopeResult } from "../diff/index.ts"; import { COUNTED_LAYERS, completeFindings, countsByRuleFile } from "../report/findings.ts"; import { endsRun, @@ -121,11 +121,14 @@ function readInside(cwd: string, file: string): string | undefined { } /** Reads each file once for a layer, however many findings point into it. */ -function sourceReader(cwd: string): (file: string) => string | undefined { +export function sourceReader( + cwd: string, + read: (cwd: string, file: string) => string | undefined = readInside, +): (file: string) => string | undefined { const cache = new Map(); return (file) => { if (!cache.has(file)) { - cache.set(file, readInside(cwd, file)); + cache.set(file, read(cwd, file)); } return cache.get(file); }; @@ -179,6 +182,10 @@ function keepArtifacts( /** The coverage a layer that does not need the tests' functions gets from the diff engine: none, without a Vitest run. */ const NO_COVERAGE: CoverageMap = new Map(); +/** The diff engine's options for a layer: a layer that does not need the tests' functions covers a changed test file with none, and the engine's own run is kept for one that does. */ +export const optionsFor = (needs: DiffNeeds, options: ScopeOptions): ScopeOptions => + needs.coveredByTests ? options : { ...options, coverageOf: () => Promise.resolve(NO_COVERAGE) }; + /** The diff, computed the first time a layer asks and kept for each kind of ask (the test-covered functions cost Vitest runs). */ function diffProvider(input: RunInput, base: string): DiffProvider { const asked = new Map>(); @@ -188,13 +195,7 @@ function diffProvider(input: RunInput, base: string): DiffProvider { return known; } const exclude = input.config.coverage.exclude.map((entry) => entry.glob); - const fresh = diffScope({ - cwd: input.cwd, - base, - exec: input.exec, - exclude, - ...(needs.coveredByTests ? {} : { coverageOf: () => Promise.resolve(NO_COVERAGE) }), - }); + const fresh = diffScope(optionsFor(needs, { cwd: input.cwd, base, exec: input.exec, exclude })); asked.set(needs.coveredByTests, fresh); return fresh; }; @@ -307,7 +308,9 @@ function producedPath(state: RunState, need: Consumed): string | undefined { /** R29 N5: the first input this layer consumes that its named producer did not leave on disk, worded for the report. */ function missingInput(layer: Layer, state: RunState): string | undefined { - const missing = layer.consumes.find((need) => producedPath(state, need) === undefined); + const missing = layer.consumes.find( + (need) => need.optional !== true && producedPath(state, need) === undefined, + ); return missing === undefined ? undefined : `input ${missing.artifact} missing: ${missing.from} ${verdictOf(state, missing.from)}`; @@ -403,8 +406,7 @@ function problemsOf(state: RunState): Problem[] { } function passedOf(state: RunState, problems: readonly Problem[]): boolean { - const clean = problems.length === 0 && state.ran === state.inRange; - return clean && !state.layers.some(ends); + return problems.length === 0 && !state.layers.some(ends); } function exitCodeOf(passed: boolean, enforced: boolean): 0 | 1 { diff --git a/src/gate/stubs.test.ts b/src/gate/stubs.test.ts index 1aece48..4151c13 100644 --- a/src/gate/stubs.test.ts +++ b/src/gate/stubs.test.ts @@ -17,12 +17,12 @@ function runReal(config: GateConfig) { }); } -const LATER = [ - ["build-output", "M4"], - ["laws", "M3"], -] as const; +const LATER = [["build-output", "M4"]] as const; -describe("G2e stubs: build-output and laws report not-run when enabled", () => { +/** The laws layer is real since M3 (G4): these tests keep it off so they judge the stubs alone. */ +const LAWS_OFF = { laws: off("the laws layer is tested in its own files") }; + +describe("G2e stubs: build-output reports not-run when enabled", () => { it("registers the seven built-in layers in gate order", () => { expect(buildRegistry(cfg()).map((layer) => layer.id)).toEqual([...BUILTIN_IDS]); }); @@ -30,12 +30,7 @@ describe("G2e stubs: build-output and laws report not-run when enabled", () => { it.each(LATER)( "enabling %s yields not-run naming %s, and a failed gate", async (id, milestone) => { - const only = Object.fromEntries( - LATER.map(([other]) => [ - other, - other === id ? { enabled: true } : off(`later milestone ${other}`), - ]), - ); + const only = { ...LAWS_OFF, [id]: { enabled: true } }; const report = await runReal( cfg({ toggles: { @@ -63,7 +58,7 @@ describe("G2e stubs: build-output and laws report not-run when enabled", () => { it.each(LATER)( "%s disabled in config is skip with the config reason, not a silent skip", async (id) => { - const toggles: Record = {}; + const toggles: Record = { ...LAWS_OFF }; for (const [other] of LATER) { toggles[other] = off(`${other} lands later`); } diff --git a/src/laws/layer/real-e2e.test.ts b/src/laws/layer/real-e2e.test.ts new file mode 100644 index 0000000..9d98547 --- /dev/null +++ b/src/laws/layer/real-e2e.test.ts @@ -0,0 +1,169 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cleanTemp, tempDir } from "../probe/testkit.ts"; + +/** + * The laws layer on a real project, with the project's own ESLint and TypeScript. The layer runs in a child process, so + * what is covered here is the process boundary and the tools, not the layer's code, which has its own tests. + */ + +afterAll(cleanTemp); + +const REPO = resolve(import.meta.dirname, "../../.."); +const ANY = "@typescript-eslint/no-explicit-any"; +const SLOW = 120_000; + +const TSCONFIG = JSON.stringify({ + compilerOptions: { + target: "es2022", + module: "nodenext", + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + noImplicitOverride: true, + noFallthroughCasesInSwitch: true, + verbatimModuleSyntax: true, + isolatedModules: true, + erasableSyntaxOnly: true, + }, + include: ["src"], +}); + +/** The config of a project that lints its TypeScript with the strict type-checked preset; `extra` is one more block. */ +function config(extra: string): string { + return [ + 'import tseslint from "typescript-eslint";', + "export default tseslint.config(", + ' { files: ["**/*.ts"], extends: [tseslint.configs.strictTypeChecked], languageOptions: { parserOptions: { projectService: true } } },', + extra, + ");", + "", + ].join("\n"); +} + +/** The script the child process runs: the real layer with the real exec and readers, over W-01 and W-04 of the catalog. */ +function layerScript(): string { + return [ + `const root = process.env.WT_REPO;`, + `const { realExec } = await import(root + "/src/gate/exec.ts");`, + `const { lawsLayer } = await import(root + "/src/laws/layer/index.ts");`, + `const { yamlCatalog } = await import(root + "/src/laws/layer/catalog.ts");`, + `const { cfg } = await import(root + "/src/gate/testkit.ts");`, + `const cwd = process.argv[1];`, + `const laws = yamlCatalog().filter((law) => law.id === "W-01" || law.id === "W-04");`, + `const result = await lawsLayer(() => laws).run({`, + ` cwd, config: cfg({ class: "ts-library" }), base: "origin/main", enforce: "hard",`, + ` env: { PATH: process.env.PATH ?? "" }, exec: realExec, outDir: cwd + "/.web-tops/out",`, + ` artifacts: new Map([["eslint-json", cwd + "/out/e.json"], ["tsc-diagnostics", cwd + "/out/t.json"]]),`, + `});`, + `process.stdout.write(JSON.stringify({ verdict: result.verdict, findings: result.findings }));`, + ].join("\n"); +} + +interface Answer { + readonly verdict: string; + readonly findings: readonly { + readonly rule: string; + readonly verdict: string; + readonly file?: string; + readonly line?: number; + readonly message: string; + }[]; +} + +function isAnswer(value: unknown): value is Answer { + return typeof value === "object" && value !== null && "verdict" in value && "findings" in value; +} + +function git(cwd: string, ...args: string[]): void { + const done = spawnSync("git", args, { cwd, encoding: "utf8" }); + if (done.status !== 0) { + throw new Error(`git ${args.join(" ")}: ${done.stderr}`); + } +} + +function project(eslintConfig: string): string { + const cwd = tempDir(); + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + mkdirSync(join(cwd, "src")); + mkdirSync(join(cwd, "out")); + writeFileSync(join(cwd, "package.json"), '{ "type": "module" }\n'); + writeFileSync(join(cwd, "tsconfig.json"), TSCONFIG); + writeFileSync(join(cwd, "src/a.ts"), "export const a: number = 1;\n"); + writeFileSync(join(cwd, "eslint.config.js"), eslintConfig); + writeFileSync(join(cwd, "out/e.json"), "[]\n"); + writeFileSync(join(cwd, "out/t.json"), "[]\n"); + git(cwd, "init", "-q"); + git(cwd, "add", "-A"); + return cwd; +} + +function judge(eslintConfig: string): Answer { + const cwd = project(eslintConfig); + const done = spawnSync("bun", ["-e", layerScript(), cwd], { + cwd, + env: { ...process.env, WT_REPO: REPO }, + encoding: "utf8", + }); + if (done.status !== 0) { + throw new Error(`the layer did not run: ${done.stderr}`); + } + const parsed: unknown = JSON.parse(done.stdout); + if (!isAnswer(parsed)) { + throw new Error(`the layer answered something else: ${done.stdout}`); + } + return parsed; +} + +const lawOf = (answer: Answer, id: string): Answer["findings"][number] | undefined => + answer.findings.find((found) => found.rule === `law/${id}`); + +describe("G4 the laws layer on a project with real ESLint and real tsc", () => { + it( + "passes W-01 and W-04 when every rule is at error for every TypeScript file and every flag is set", + () => { + const answer = judge(config("")); + expect(lawOf(answer, "W-01")?.verdict).toBe("pass"); + expect(lawOf(answer, "W-04")?.verdict).toBe("pass"); + expect(answer.verdict).toBe("pass"); + }, + SLOW, + ); + + it.each([ + ["off", "off"], + ["warn", "warn"], + ])( + "refuses W-01 at the config line of an override that sets no-explicit-any to %s", + (_name, level) => { + const override = ` { files: ["src/**"], rules: { "${ANY}": "${level}" } },`; + const text = config(override); + const line = text.split("\n").indexOf(override) + 1; + const answer = judge(text); + expect(lawOf(answer, "W-01")?.verdict).toBe("refuse"); + expect(answer.verdict).toBe("refuse"); + const spot = answer.findings.find((found) => found.rule === ANY); + expect(spot).toMatchObject({ verdict: "refuse", file: "eslint.config.js", line }); + expect(spot?.message).toBe(`${ANY} is below error for 1 file(s), such as src/a.ts`); + expect(lawOf(answer, "W-04")?.verdict).toBe("pass"); + }, + SLOW, + ); + + it( + "says why W-01 is not evaluated when the config reaches no TypeScript file, and refuses the file-set", + () => { + const answer = judge(`export default [{ rules: { "${ANY}": "error" } }];\n`); + const reading = answer.findings.find((found) => found.rule === "law/W-01"); + expect(reading?.verdict).toBe("not-evaluated"); + expect(reading?.message).toContain("ESLint's config reaches no TypeScript file"); + expect(answer.findings.find((found) => found.rule === "file-set")).toMatchObject({ + verdict: "refuse", + file: "src/a.ts", + }); + }, + SLOW, + ); +}); diff --git a/src/laws/readers/eslint-judge.test.ts b/src/laws/readers/eslint-judge.test.ts index 5788ed7..2050fcc 100644 --- a/src/laws/readers/eslint-judge.test.ts +++ b/src/laws/readers/eslint-judge.test.ts @@ -153,7 +153,8 @@ describe("G4a the ESLint reader judges a law by each file's effective config", ( expect(reading).toEqual({ law: "W-01", verdict: "not-evaluated", - reason: "no TypeScript file was linted", + reason: + "no TypeScript file was linted: ESLint's config reaches no TypeScript file (no config block has files for **/*.ts and the TypeScript parser, or an ignore drops them), so no rule can be judged", findings: [], }); }); diff --git a/src/laws/readers/eslint-judge.ts b/src/laws/readers/eslint-judge.ts index bcf5f57..9d3d8a9 100644 --- a/src/laws/readers/eslint-judge.ts +++ b/src/laws/readers/eslint-judge.ts @@ -160,11 +160,16 @@ function countedReading( : undefined; } +/** Why no rule can be judged, worded for an adopter whose ESLint config cannot reach `.ts` files at all. */ +function unlintedReason(): string { + return "no TypeScript file was linted: ESLint's config reaches no TypeScript file (no config block has files for **/*.ts and the TypeScript parser, or an ignore drops them), so no rule can be judged"; +} + /** No TypeScript file was linted: the typed rules have no file to be judged over. */ function unjudgedReading(law: string, evidence: Evidence): LawReading | undefined { return [...evidence.levels.keys()].some(isTypeScript) ? undefined - : { law, verdict: "not-evaluated", reason: "no TypeScript file was linted", findings: [] }; + : { law, verdict: "not-evaluated", reason: unlintedReason(), findings: [] }; } function passReading(law: string, counted: readonly string[]): LawReading { diff --git a/src/layers/tests/launch-label.test.ts b/src/layers/tests/launch-label.test.ts new file mode 100644 index 0000000..a0b42e8 --- /dev/null +++ b/src/layers/tests/launch-label.test.ts @@ -0,0 +1,52 @@ +import { symlinkSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cfg } from "../../gate/testkit.ts"; +import type { Exec, LayerContext } from "../../gate/types.ts"; +import { cleanTemp, tempDir } from "../../laws/probe/testkit.ts"; +import { runTool } from "../hygiene/tooling.ts"; +import { testsLayer } from "./index.ts"; + +afterAll(cleanTemp); + +const REPO = resolve(import.meta.dirname, "../../.."); + +function context(exec: Exec, installed = true): LayerContext { + const cwd = tempDir(); + if (installed) { + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + } + return { + cwd, + config: cfg(), + base: "origin/main", + enforce: "hard", + env: {}, + exec, + outDir: join(cwd, ".web-tops", "out"), + artifacts: new Map(), + }; +} + +describe("gate 2 says which tool it could not run", () => { + it("names vitest when it did not finish in time and was killed", async () => { + const exec: Exec = () => + Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-timeout" }); + const result = await testsLayer.run(context(exec)); + expect(result.verdict).toBe("fail"); + expect(result.findings).toMatchObject([ + { rule: "tool-timeout", message: "vitest did not finish in time and was killed" }, + ]); + }); + + it("is tool-missing, with nothing on stdout, when vitest is not installed in the project", async () => { + const exec: Exec = () => Promise.reject(new Error("nothing is started")); + const result = await runTool(context(exec, false), "vitest", []); + expect(result).toEqual({ + code: null, + stdout: "", + stderr: "vitest is not installed in node_modules (package vitest)", + failure: "tool-missing", + }); + }); +}); diff --git a/src/report/findings-edges.test.ts b/src/report/findings-edges.test.ts new file mode 100644 index 0000000..81cd56f --- /dev/null +++ b/src/report/findings-edges.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "vitest"; +import { completeFindings, fingerprint, type RawFinding } from "./findings.ts"; + +/** What makes a fingerprint: the parts it hashes, the order among repeats, and which findings repeat each other. */ + +const found = (extra: Partial = {}): RawFinding => ({ + tool: "eslint", + rule: "no-console", + verdict: "fail", + message: "no console", + file: "src/a.ts", + ...extra, +}); + +const noSource = (): undefined => undefined; + +describe("a fingerprint", () => { + it("is fp1 and 16 hex characters of the sha256 of its parts, joined by NUL", () => { + const parts = { + tool: "eslint", + rule: "no-console", + file: "src/a.ts", + anchor: "x", + occurrence: 1, + }; + expect(fingerprint(parts)).toBe("fp1:f60399d1c94096d8"); + expect(fingerprint({ ...parts, file: null })).toBe("fp1:ce4ebab2c81aa1e8"); + expect(fingerprint({ ...parts, occurrence: 2 })).not.toBe(fingerprint(parts)); + }); +}); + +describe("the order among repeats", () => { + const fp = (rule: string, file: string | null, anchor: string, occurrence: number): string => + fingerprint({ tool: "eslint", rule, file, anchor, occurrence }); + + it("counts each repeat by line, whatever order the tool listed them", () => { + const lines = [5, 2, 9, 2]; + const raws = lines.map((line) => found({ line })); + const done = completeFindings(raws, noSource); + const expected = [3, 1, 4, 2].map((n) => fp("no-console", "src/a.ts", "no console", n)); + expect(done.map((item) => item.fingerprint)).toEqual(expected); + }); + + it("counts repeats already in line order as they are", () => { + const done = completeFindings( + [1, 2, 3].map((line) => found({ line })), + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual( + [1, 2, 3].map((n) => fp("no-console", "src/a.ts", "no console", n)), + ); + }); + + it("counts repeats within one tool, rule, file and anchor, and no wider", () => { + const done = completeFindings( + [ + found({ rule: "a" }), + found({ rule: "b" }), + found({ rule: "a", file: "src/b.ts" }), + found({ rule: "a", message: "other" }), + found({ rule: "a" }), + ], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("a", "src/a.ts", "no console", 1), + fp("b", "src/a.ts", "no console", 1), + fp("a", "src/b.ts", "no console", 1), + fp("a", "src/a.ts", "other", 1), + fp("a", "src/a.ts", "no console", 2), + ]); + }); + + it("keeps the parts of the key apart: a rule and a file that read the same joined are not repeats", () => { + const done = completeFindings( + [found({ rule: "ab", file: "c" }), found({ rule: "a", file: "bc" })], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("ab", "c", "no console", 1), + fp("a", "bc", "no console", 1), + ]); + }); + + it("treats no file as an empty one, and a file named like the placeholder as its own", () => { + const none: RawFinding = { tool: "eslint", rule: "r", verdict: "fail", message: "m" }; + const done = completeFindings( + [ + none, + found({ rule: "r", file: "", message: "m" }), + found({ rule: "r", file: "x", message: "m" }), + ], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("r", null, "m", 1), + fp("r", "", "m", 2), + fp("r", "x", "m", 1), + ]); + }); +}); + +describe("what a completed finding carries beyond the shape", () => { + it("has required, ack and survivor only when the finding had them", () => { + const plain = completeFindings([found()], noSource)[0]; + expect(Object.keys(plain ?? {}).sort()).toEqual([ + "file", + "fingerprint", + "law", + "line", + "message", + "rule", + "tool", + "verdict", + ]); + const survivor = { + file: "src/a.ts", + function: "f", + mutator: "m", + original: "o", + replacement: "r", + occurrence: 1, + }; + const full = completeFindings( + [found({ required: false, survivor: { ...survivor }, ack: undefined })], + noSource, + )[0]; + expect(full).toMatchObject({ required: false, survivor }); + expect("ack" in (full ?? {})).toBe(false); + }); +}); diff --git a/src/report/findings.ts b/src/report/findings.ts index e712eb7..34aa8fb 100644 --- a/src/report/findings.ts +++ b/src/report/findings.ts @@ -62,14 +62,20 @@ function lineAt(raws: readonly RawFinding[], index: number): number { } function byLine(raws: readonly RawFinding[]): (left: number, right: number) => number { - return (left, right) => lineAt(raws, left) - lineAt(raws, right) || left - right; + return (left, right) => lineAt(raws, left) - lineAt(raws, right); +} + +interface Anchored { + readonly raw: RawFinding; + readonly anchor: string; } /** Each finding's order among its repeats, by line and then by the order the tool listed them. */ -function occurrences(raws: readonly RawFinding[], anchors: readonly string[]): number[] { +function occurrences(found: readonly Anchored[]): number[] { + const raws = found.map((item) => item.raw); const groups = new Map(); - raws.forEach((raw, index) => { - const key = groupKey(raw, anchors[index] ?? ""); + found.forEach(({ raw, anchor }, index) => { + const key = groupKey(raw, anchor); groups.set(key, [...(groups.get(key) ?? []), index]); }); const order = raws.map(() => 1); @@ -118,7 +124,7 @@ function complete(raw: RawFinding, anchor: string, occurrence: number): Finding * anchored on the flagged line (WEB_TOPS.md section 6.6). `read` returns a file's text, or undefined. */ export function completeFindings(raws: readonly RawFinding[], read: Reader): Finding[] { - const anchors = raws.map((raw) => anchorFor(raw, read)); - const order = occurrences(raws, anchors); - return raws.map((raw, index) => complete(raw, anchors[index] ?? "", order[index] ?? 1)); + const found = raws.map((raw) => ({ raw, anchor: anchorFor(raw, read) })); + const order = occurrences(found); + return found.map(({ raw, anchor }, index) => complete(raw, anchor, order[index] ?? 1)); }