From 335717a2e0db9066bbb2461eb53f2178b13c6c7a Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 04:35:50 -0500 Subject: [PATCH 1/8] Gate runner: let a layer declare an input optional, and register the laws layer A layer input marked optional no longer makes the layer not-run when its producer left no file: the runner passes it when it is there and the layer says what a missing one means. The registry now holds the laws layer in place of its stub, read from laws/W-NN.yaml. The hand-off test keeps the rule that a required input is declared by its producer, and the stub tests keep the laws layer off so they judge the build-output stub alone. Co-Authored-By: Claude Sonnet 5.5 --- src/gate/handoffs.test.ts | 21 ++++++++++++++------- src/gate/registry.ts | 6 ++++-- src/gate/runner.ts | 4 +++- src/gate/stubs.test.ts | 19 +++++++------------ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/gate/handoffs.test.ts b/src/gate/handoffs.test.ts index 23312ca..9bb5678 100644 --- a/src/gate/handoffs.test.ts +++ b/src/gate/handoffs.test.ts @@ -5,7 +5,7 @@ import type { Report } from "../report/types.ts"; import { runGate } from "./runner.ts"; import { buildRegistry } from "./registry.ts"; import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; -import type { GateConfig, GateOptions, Layer } from "./types.ts"; +import type { Consumed, GateConfig, GateOptions, Layer } from "./types.ts"; /** R29 N5: layers declare the artifacts they produce and consume; a missing input is `not-run`, a present one is judged. */ @@ -307,12 +307,19 @@ describe("G2c the real registry declares its hand-offs", () => { const layers = buildRegistry(cfg()); const byId = new Map(layers.map((layer) => [layer.id, layer])); - it("never consumes from a layer it does not require, and the producer declares what is consumed", () => { - for (const layer of layers) { - for (const need of layer.consumes) { - expect(layer.requires).toContain(need.from); - expect(byId.get(need.from)?.produces).toContain(need.artifact); - } + const isRequired = (need: Consumed): boolean => need.optional !== true; + + it("never consumes from a layer it does not require", () => { + const needs = layers.flatMap((layer) => layer.consumes.map((need) => [layer, need] as const)); + for (const [layer, need] of needs) { + expect(layer.requires).toContain(need.from); + } + }); + + it("the producer declares what a layer consumes, but an optional input may be one it does not write (R92)", () => { + const needs = layers.flatMap((layer) => layer.consumes.filter(isRequired)); + for (const need of needs) { + expect(byId.get(need.from)?.produces).toContain(need.artifact); } }); diff --git a/src/gate/registry.ts b/src/gate/registry.ts index 90eeed0..1ba0206 100644 --- a/src/gate/registry.ts +++ b/src/gate/registry.ts @@ -1,10 +1,12 @@ import { coverageLayer } from "../layers/coverage/index.ts"; import { crapLayer } from "../layers/crap/index.ts"; +import { yamlCatalog } from "../laws/layer/catalog.ts"; +import { lawsLayer } from "../laws/layer/index.ts"; import { mutationLayer } from "../layers/mutation/index.ts"; import { hygieneLayer } from "../layers/hygiene/index.ts"; import { testsLayer } from "../layers/tests/index.ts"; import { compareGates } from "./order.ts"; -import { BUILD_OUTPUT_STUB, customLayer, LAWS_STUB } from "./stubs.ts"; +import { BUILD_OUTPUT_STUB, customLayer } from "./stubs.ts"; import type { GateConfig, Layer } from "./types.ts"; /** The layers of one run: the seven built-ins and the config's custom layers, in gate order. */ @@ -18,6 +20,6 @@ export function buildRegistry(config: GateConfig): Layer[] { mutationLayer, BUILD_OUTPUT_STUB, ...custom, - LAWS_STUB, + lawsLayer(yamlCatalog), ].sort((a, b) => compareGates(a.gate, b.gate)); } diff --git a/src/gate/runner.ts b/src/gate/runner.ts index ddf7a95..86e2247 100644 --- a/src/gate/runner.ts +++ b/src/gate/runner.ts @@ -307,7 +307,9 @@ function producedPath(state: RunState, need: Consumed): string | undefined { /** R29 N5: the first input this layer consumes that its named producer did not leave on disk, worded for the report. */ function missingInput(layer: Layer, state: RunState): string | undefined { - const missing = layer.consumes.find((need) => producedPath(state, need) === undefined); + const missing = layer.consumes.find( + (need) => need.optional !== true && producedPath(state, need) === undefined, + ); return missing === undefined ? undefined : `input ${missing.artifact} missing: ${missing.from} ${verdictOf(state, missing.from)}`; diff --git a/src/gate/stubs.test.ts b/src/gate/stubs.test.ts index 1aece48..4151c13 100644 --- a/src/gate/stubs.test.ts +++ b/src/gate/stubs.test.ts @@ -17,12 +17,12 @@ function runReal(config: GateConfig) { }); } -const LATER = [ - ["build-output", "M4"], - ["laws", "M3"], -] as const; +const LATER = [["build-output", "M4"]] as const; -describe("G2e stubs: build-output and laws report not-run when enabled", () => { +/** The laws layer is real since M3 (G4): these tests keep it off so they judge the stubs alone. */ +const LAWS_OFF = { laws: off("the laws layer is tested in its own files") }; + +describe("G2e stubs: build-output reports not-run when enabled", () => { it("registers the seven built-in layers in gate order", () => { expect(buildRegistry(cfg()).map((layer) => layer.id)).toEqual([...BUILTIN_IDS]); }); @@ -30,12 +30,7 @@ describe("G2e stubs: build-output and laws report not-run when enabled", () => { it.each(LATER)( "enabling %s yields not-run naming %s, and a failed gate", async (id, milestone) => { - const only = Object.fromEntries( - LATER.map(([other]) => [ - other, - other === id ? { enabled: true } : off(`later milestone ${other}`), - ]), - ); + const only = { ...LAWS_OFF, [id]: { enabled: true } }; const report = await runReal( cfg({ toggles: { @@ -63,7 +58,7 @@ describe("G2e stubs: build-output and laws report not-run when enabled", () => { it.each(LATER)( "%s disabled in config is skip with the config reason, not a silent skip", async (id) => { - const toggles: Record = {}; + const toggles: Record = { ...LAWS_OFF }; for (const [other] of LATER) { toggles[other] = off(`${other} lands later`); } From 3e8c19afb7eeddef73645e670e8f544f0f203f00 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 04:54:59 -0500 Subject: [PATCH 2/8] Gate runner: test what the runner, plan, registry and finding fingerprints do at their edges Registering the laws layer and the optional input put the runner, the plan, the registry, the finding fingerprints, the ratchet mode check and the tests layer launch message in the scope of the mutation run, and 39 of their mutants survived. Tests now pin each: what a layer entry holds when nothing ran, a layer that throws, the clock, the producer named in a missing input, an optional input present and absent, the laws layer after a stop, the report crap scope, a switch with no reason, layers that require each other, an id nothing registers, a fingerprint against an independent sha256, the order among repeats, and git as the only command for a diff the tests do not need. Where a mutant could not be told from the original, the code is simpler: no tie rule in the layer lookup, one anchored list instead of two parallel ones, no sort of a list already in gate order, no second clause that the problem list already says, and a file reader that can be handed in so the once-per-file cache can be counted. Co-Authored-By: Claude Sonnet 5.5 --- src/gate/diff-wiring.test.ts | 32 +++++ src/gate/plan-edges.test.ts | 71 +++++++++ src/gate/plan.ts | 2 +- src/gate/ratchet.test.ts | 9 ++ src/gate/registry-order.test.ts | 69 +++++++++ src/gate/registry.ts | 3 +- src/gate/runner-edges.test.ts | 231 ++++++++++++++++++++++++++++++ src/gate/runner.ts | 10 +- src/layers/tests/tests.test.ts | 11 ++ src/report/findings-edges.test.ts | 121 ++++++++++++++++ src/report/findings.ts | 18 ++- 11 files changed, 564 insertions(+), 13 deletions(-) create mode 100644 src/gate/plan-edges.test.ts create mode 100644 src/gate/registry-order.test.ts create mode 100644 src/gate/runner-edges.test.ts create mode 100644 src/report/findings-edges.test.ts diff --git a/src/gate/diff-wiring.test.ts b/src/gate/diff-wiring.test.ts index 16111fb..85f03e5 100644 --- a/src/gate/diff-wiring.test.ts +++ b/src/gate/diff-wiring.test.ts @@ -130,6 +130,38 @@ describe("R63 the default CRAP scope is the diff, computed by the gate", () => { }); }); +describe("R63 a layer that does not need the tests' functions starts no test run", () => { + it("runs git and nothing else for a diff that changes a test file", async () => { + const dir = repoChanging(true); + write(dir, { "src/a.test.ts": "export const t = 1;\n" }); + commit(dir, "a test"); + const commands: string[] = []; + const exec: Exec = (command, args, options) => { + commands.push(command); + return realExec(command, args, options); + }; + const asking: Layer = { + ...fakeLayer({ id: "hygiene", gate: 1 }), + run: async (ctx: LayerContext) => { + await ctx.diff?.({ coveredByTests: false }); + return { verdict: "pass", findings: [], artifacts: {} }; + }, + }; + await runGate({ + layers: [asking], + config: cfg(), + options: opts({ base: "main" }), + cwd: dir, + env: {}, + exec, + outDir: join(dir, ".web-tops", "out"), + now: tickingClock(), + }); + expect(commands.length).toBeGreaterThan(0); + expect(new Set(commands)).toEqual(new Set(["git"])); + }); +}); + describe("R63 the provider computes the diff once for each kind of request", () => { /** How many git commands a run makes when its one layer asks for the diff as `asks` says. */ async function gitCalls(asks: readonly DiffNeeds[]): Promise { diff --git a/src/gate/plan-edges.test.ts b/src/gate/plan-edges.test.ts new file mode 100644 index 0000000..1f0e294 --- /dev/null +++ b/src/gate/plan-edges.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { planGate } from "./plan.ts"; +import { cfg, fakeLayer, opts } from "./testkit.ts"; +import type { GateConfig } from "./types.ts"; + +/** The plan's edges: a switch with no reason, layers that require each other, and ids nothing registers. */ + +const enabled = (id: string): GateConfig => cfg({ toggles: { [id]: { enabled: true } } }); + +describe("the reason a layer is skipped", () => { + it("is the config's own words, and says so when the config gave none", () => { + const layers = [fakeLayer({ id: "hygiene" })]; + const bare = cfg({ toggles: { hygiene: { enabled: false } } }); + expect(planGate(layers, bare, opts())[0]?.skip).toBe("disabled in web-tops.yaml"); + const said = cfg({ toggles: { hygiene: { enabled: false, reason: "lands later" } } }); + expect(planGate(layers, said, opts())[0]?.skip).toBe("lands later"); + }); +}); + +describe("--only follows what a layer requires", () => { + it("ends when two layers require each other", () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, requires: ["tests"] }), + fakeLayer({ id: "tests", gate: 2, requires: ["hygiene"] }), + fakeLayer({ id: "coverage", gate: 3 }), + ]; + const plan = planGate(layers, cfg(), opts({ only: "hygiene" })); + expect(plan.map((item) => [item.id, item.skip, item.requiredBy])).toEqual([ + ["hygiene", undefined, undefined], + ["tests", undefined, "hygiene"], + ["coverage", "not selected by --only hygiene", undefined], + ]); + }); + + it("pulls a layer in once, by the first that needs it", () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1 }), + fakeLayer({ id: "tests", gate: 2, requires: ["hygiene"] }), + fakeLayer({ id: "coverage", gate: 3, requires: ["tests", "hygiene"] }), + ]; + const plan = planGate(layers, cfg(), opts({ only: "coverage" })); + expect(plan.map((item) => item.requiredBy)).toEqual(["coverage", "coverage", undefined]); + }); + + it("names a layer nothing registers and pulls nothing else in", () => { + const layers = [fakeLayer({ id: "hygiene", gate: 1 })]; + const plan = planGate(layers, cfg(), opts({ only: "ghost" })); + expect(plan.map((item) => [item.id, item.skip])).toEqual([ + ["hygiene", "not selected by --only ghost"], + ]); + }); +}); + +describe("an id the config enables and nothing registers", () => { + const layers = [fakeLayer({ id: "hygiene", gate: 1 })]; + + it("is in the plan as a layer of its own, and not when --only picks a layer", () => { + expect(planGate(layers, enabled("ghost"), opts()).map((item) => item.id)).toEqual([ + "hygiene", + "ghost", + ]); + expect( + planGate(layers, enabled("ghost"), opts({ only: "hygiene" })).map((item) => item.id), + ).toEqual(["hygiene"]); + }); + + it("is skipped for the reason any layer would be, such as a tier above the one asked", () => { + const plan = planGate(layers, enabled("ghost"), opts({ maxTier: 0 })); + expect(plan.find((item) => item.id === "ghost")?.skip).toBe("tier 1 is above --max-tier 0"); + }); +}); diff --git a/src/gate/plan.ts b/src/gate/plan.ts index 7e80d02..0b6799a 100644 --- a/src/gate/plan.ts +++ b/src/gate/plan.ts @@ -33,7 +33,7 @@ interface Scope { } function requiresOf(layers: readonly Layer[], id: string): readonly string[] { - return layers.find((layer) => layer.id === id)?.requires ?? []; + return layers.filter((layer) => layer.id === id).flatMap((layer) => layer.requires); } function select( diff --git a/src/gate/ratchet.test.ts b/src/gate/ratchet.test.ts index 2d9ca61..5318558 100644 --- a/src/gate/ratchet.test.ts +++ b/src/gate/ratchet.test.ts @@ -90,6 +90,15 @@ function layer(report: Report, id: string): Report["layers"][number] { return found; } +describe("G9b: only the ratchet mode reads a baseline", () => { + it("leaves a run above its baseline alone under hard, though the config names one", async () => { + const report = await run(project(BASE), layersWith(crapRows(9)), ratchet({ enforce: "hard" })); + expect(layer(report, "crap").verdict).toBe("pass"); + expect(layer(report, "crap").findings).toEqual([]); + expect(report.passed).toBe(true); + }); +}); + describe("G9b: a run above its baseline is refused, layer by layer", () => { it("passes a run that equals its baseline, and one that is below it", async () => { const same = await run( diff --git a/src/gate/registry-order.test.ts b/src/gate/registry-order.test.ts new file mode 100644 index 0000000..eb6e81e --- /dev/null +++ b/src/gate/registry-order.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { buildRegistry } from "./registry.ts"; +import { compareGates } from "./order.ts"; +import { customLayer } from "./stubs.ts"; +import { cfg, noExec } from "./testkit.ts"; +import type { LayerContext } from "./types.ts"; + +describe("the registry lists the layers in gate order", () => { + it("with the config's custom layers between gate 6 and laws", () => { + const config = cfg({ + custom: [ + { id: "extra", tier: 1, command: "echo" }, + { id: "more", tier: 2 }, + ], + }); + const layers = buildRegistry(config); + expect(layers.map((layer) => layer.id)).toEqual([ + "hygiene", + "tests", + "coverage", + "crap", + "mutation", + "build-output", + "extra", + "more", + "laws", + ]); + const gates = layers.map((layer) => layer.gate); + expect(gates).toEqual([...gates].sort(compareGates)); + }); +}); + +describe("a custom layer is a gate 6 layer that is not run yet", () => { + it("needs and gives nothing, for every launch class, and is not-run with the reason it has no command", async () => { + const mapped = customLayer("extra", 2, "echo"); + expect(mapped).toMatchObject({ + id: "extra", + gate: 6, + tier: 2, + requires: [], + produces: [], + consumes: [], + }); + expect(mapped.classes).toEqual([ + "ts-library", + "ui-components", + "astro-site", + "web-app", + "node-service", + ]); + const ctx: LayerContext = { + cwd: ".", + config: cfg(), + base: "origin/main", + enforce: "hard", + env: {}, + exec: noExec, + outDir: ".", + artifacts: new Map(), + }; + const results = [await mapped.run(ctx), await customLayer("bare", 1, undefined).run(ctx)]; + expect(results.map((result) => result.verdict)).toEqual(["not-run", "not-run"]); + expect(results.map((result) => result.reason)).toEqual([ + "custom layers run from M4", + "no command is mapped to this layer", + ]); + expect(results.map((result) => result.findings)).toEqual([[], []]); + }); +}); diff --git a/src/gate/registry.ts b/src/gate/registry.ts index 1ba0206..4740f41 100644 --- a/src/gate/registry.ts +++ b/src/gate/registry.ts @@ -5,7 +5,6 @@ import { lawsLayer } from "../laws/layer/index.ts"; import { mutationLayer } from "../layers/mutation/index.ts"; import { hygieneLayer } from "../layers/hygiene/index.ts"; import { testsLayer } from "../layers/tests/index.ts"; -import { compareGates } from "./order.ts"; import { BUILD_OUTPUT_STUB, customLayer } from "./stubs.ts"; import type { GateConfig, Layer } from "./types.ts"; @@ -21,5 +20,5 @@ export function buildRegistry(config: GateConfig): Layer[] { BUILD_OUTPUT_STUB, ...custom, lawsLayer(yamlCatalog), - ].sort((a, b) => compareGates(a.gate, b.gate)); + ]; } diff --git a/src/gate/runner-edges.test.ts b/src/gate/runner-edges.test.ts new file mode 100644 index 0000000..b3ac960 --- /dev/null +++ b/src/gate/runner-edges.test.ts @@ -0,0 +1,231 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Report } from "../report/types.ts"; +import { runGate, sourceReader } from "./runner.ts"; +import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; +import type { GateConfig, GateOptions, Layer, LayerContext } from "./types.ts"; + +/** The runner's edges: what a layer entry holds when nothing ran, when a layer threw, and what each input and stop does. */ + +const projects: ReturnType[] = []; + +afterEach(() => { + for (const project of projects.splice(0)) { + project.cleanup(); + } +}); + +function setup(): { dir: string; write: (name: string) => string; run: Run } { + const project = tempProject(); + projects.push(project); + const outDir = join(project.dir, ".web-tops", "out"); + return { + dir: project.dir, + write: (name) => { + const path = join(outDir, name); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "{}\n"); + return path; + }, + run: (layers, config = cfg(), options = opts()) => + runGate({ + layers, + config, + options, + cwd: project.dir, + env: {}, + exec: noExec, + outDir, + now: tickingClock(), + }), + }; +} + +type Run = ( + layers: readonly Layer[], + config?: GateConfig, + options?: GateOptions, +) => Promise; + +function entry(report: Report, id: string): Report["layers"][number] { + const found = report.layers.find((layer) => layer.id === id); + if (found === undefined) { + throw new Error(`no ${id} in the report`); + } + return found; +} + +describe("the entry of a layer that did not run", () => { + it("has no findings, metrics or artifacts and took no time", async () => { + const config = cfg({ toggles: { hygiene: off("not for this project") } }); + const report = await setup().run([fakeLayer({ id: "hygiene" })], config); + const hygiene = entry(report, "hygiene"); + expect(hygiene.verdict).toBe("skip"); + expect(hygiene.findings).toStrictEqual([]); + expect(hygiene.metrics).toStrictEqual({}); + expect(hygiene.artifacts).toStrictEqual([]); + expect(hygiene.duration_ms).toBe(0); + }); +}); + +describe("a layer that throws", () => { + it("fails with what it threw, as a finding of its own, in the time it took", async () => { + const report = await setup().run([fakeLayer({ id: "hygiene", throws: new Error("boom") })]); + const hygiene = entry(report, "hygiene"); + expect(hygiene.verdict).toBe("fail"); + expect(hygiene.reason).toBe("layer threw: boom"); + expect(hygiene.findings).toHaveLength(1); + expect(hygiene.findings[0]).toMatchObject({ + tool: "webtops", + rule: "layer-threw", + verdict: "fail", + message: "boom", + }); + expect(hygiene.duration_ms).toBeGreaterThan(0); + expect(hygiene.duration_ms).toBeLessThan(100); + }); + + it("names what is thrown when it is not an Error", async () => { + const thrower: Layer = { + ...fakeLayer({ id: "hygiene" }), + run: () => Promise.reject(new Error("plain")), + }; + const report = await setup().run([thrower]); + expect(entry(report, "hygiene").reason).toBe("layer threw: plain"); + }); +}); + +describe("the time a layer took", () => { + it("is what the clock advanced while it ran", async () => { + const report = await setup().run([fakeLayer({ id: "hygiene" })]); + const took = entry(report, "hygiene").duration_ms; + expect(took).toBeGreaterThan(0); + expect(took).toBeLessThan(100); + }); +}); + +describe("a missing input names the layer that should have produced it", () => { + it("is the producer's own verdict, not the first layer's", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1 }), + fakeLayer({ id: "tests", gate: 2, verdict: "fail", reason: "a test failed" }), + fakeLayer({ + id: "coverage", + gate: 3, + requires: ["tests"], + consumes: [{ artifact: "coverage-json", from: "tests" }], + }), + ]; + const report = await setup().run(layers, cfg(), opts({ keepGoing: true })); + expect(entry(report, "coverage").reason).toBe("input coverage-json missing: tests fail"); + }); +}); + +/** A layer that records the inputs the runner showed it. */ +function watching(seen: string[][], consumes: Layer["consumes"]): Layer { + return { + ...fakeLayer({ id: "coverage", gate: 3, requires: ["hygiene"], consumes }), + run: (ctx: LayerContext) => { + seen.push([...ctx.artifacts.keys()]); + return Promise.resolve({ verdict: "pass", findings: [], artifacts: {} }); + }, + }; +} + +describe("R92 an optional input", () => { + const optional = [{ artifact: "audit-json", from: "hygiene", optional: true }]; + + it("is passed when the producer left it, and the layer runs without it when not", async () => { + const seen: string[][] = []; + const home = setup(); + const left = fakeLayer({ + id: "hygiene", + artifacts: { "audit-json": home.write("hygiene/audit-json.json") }, + }); + const withIt = await home.run([left, watching(seen, optional)]); + expect(entry(withIt, "coverage").verdict).toBe("pass"); + const without = await setup().run([fakeLayer({ id: "hygiene" }), watching(seen, optional)]); + expect(entry(without, "coverage").verdict).toBe("pass"); + expect(seen).toEqual([["audit-json"], []]); + }); + + it("is not a missing input, while a required one beside it still is", async () => { + const seen: string[][] = []; + const both = [...optional, { artifact: "tsc-diagnostics", from: "hygiene" }]; + const report = await setup().run([fakeLayer({ id: "hygiene" }), watching(seen, both)]); + expect(entry(report, "coverage")).toMatchObject({ + verdict: "not-run", + reason: "input tsc-diagnostics missing: hygiene pass", + }); + expect(seen).toEqual([]); + }); +}); + +describe("after a stop, the laws layer judges only when every layer it requires has run", () => { + it("is not run when one of them was stopped before it ran", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "coverage", gate: 3 }), + fakeLayer({ id: "laws", gate: "laws", requires: ["hygiene", "coverage"] }), + ]; + const report = await setup().run(layers); + expect(entry(report, "laws")).toMatchObject({ + verdict: "not-run", + reason: "stopped: gate 1 failed", + }); + }); + + it("judges when the one it requires has run, whatever its verdict", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "coverage", gate: 3 }), + fakeLayer({ id: "laws", gate: "laws", requires: ["hygiene"] }), + ]; + const report = await setup().run(layers); + expect(entry(report, "laws").verdict).toBe("pass"); + expect(entry(report, "coverage").verdict).toBe("not-run"); + }); +}); + +describe("a run that stopped on a failing gate", () => { + it("fails with no problem listed: the layers it never reached are named by their own entries", async () => { + const layers = [ + fakeLayer({ id: "hygiene", gate: 1, verdict: "fail", reason: "lint failed" }), + fakeLayer({ id: "tests", gate: 2 }), + ]; + const report = await setup().run(layers); + expect(report.problems).toEqual([]); + expect(report.passed).toBe(false); + expect(entry(report, "tests").verdict).toBe("not-run"); + }); +}); + +describe("the CRAP scope the report names", () => { + const all = cfg({ crap: { cap: 4, scope: "all", nullishDiscount: true } }); + + it("is diff by default, and all from the config or --all", async () => { + const layers = [fakeLayer({ id: "hygiene" })]; + expect((await setup().run(layers)).crap_scope).toBe("diff"); + expect((await setup().run(layers, all)).crap_scope).toBe("all"); + expect((await setup().run(layers, cfg(), opts({ all: true }))).crap_scope).toBe("all"); + }); +}); + +describe("the text of a file a finding points at", () => { + it("is read once however many findings point into it, and by file", () => { + const reads: string[] = []; + const read = sourceReader("/work", (cwd, file) => { + reads.push(`${cwd}:${file}`); + return file === "a.ts" ? "text" : undefined; + }); + expect([read("a.ts"), read("a.ts"), read("b.ts"), read("b.ts"), read("a.ts")]).toEqual([ + "text", + "text", + undefined, + undefined, + "text", + ]); + expect(reads).toEqual(["/work:a.ts", "/work:b.ts"]); + }); +}); diff --git a/src/gate/runner.ts b/src/gate/runner.ts index 86e2247..9cb1ee1 100644 --- a/src/gate/runner.ts +++ b/src/gate/runner.ts @@ -121,11 +121,14 @@ function readInside(cwd: string, file: string): string | undefined { } /** Reads each file once for a layer, however many findings point into it. */ -function sourceReader(cwd: string): (file: string) => string | undefined { +export function sourceReader( + cwd: string, + read: (cwd: string, file: string) => string | undefined = readInside, +): (file: string) => string | undefined { const cache = new Map(); return (file) => { if (!cache.has(file)) { - cache.set(file, readInside(cwd, file)); + cache.set(file, read(cwd, file)); } return cache.get(file); }; @@ -405,8 +408,7 @@ function problemsOf(state: RunState): Problem[] { } function passedOf(state: RunState, problems: readonly Problem[]): boolean { - const clean = problems.length === 0 && state.ran === state.inRange; - return clean && !state.layers.some(ends); + return problems.length === 0 && !state.layers.some(ends); } function exitCodeOf(passed: boolean, enforced: boolean): 0 | 1 { diff --git a/src/layers/tests/tests.test.ts b/src/layers/tests/tests.test.ts index 959602a..fba5aa0 100644 --- a/src/layers/tests/tests.test.ts +++ b/src/layers/tests/tests.test.ts @@ -313,6 +313,17 @@ describe("G4a tests: zero tests, .only and .skip each fail", () => { expect(rulesOf(result)).toEqual([rule]); }); + it("names vitest in what it says when node is missing or vitest is killed", async () => { + const missing = await runTests(real({}), () => + Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-missing" as const }), + ); + expect(missing.findings[0]?.message).toBe("node is not on PATH, so vitest cannot run"); + const slow = await runTests(real({}), () => + Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-timeout" as const }), + ); + expect(slow.findings[0]?.message).toBe("vitest did not finish in time and was killed"); + }); + it("results that are not JSON or not an object are findings, not passes", async () => { const item = real({}); const writes = diff --git a/src/report/findings-edges.test.ts b/src/report/findings-edges.test.ts new file mode 100644 index 0000000..fe694b3 --- /dev/null +++ b/src/report/findings-edges.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from "vitest"; +import { completeFindings, fingerprint, type RawFinding } from "./findings.ts"; + +/** What makes a fingerprint: the parts it hashes, the order among repeats, and which findings repeat each other. */ + +const found = (extra: Partial = {}): RawFinding => ({ + tool: "eslint", + rule: "no-console", + verdict: "fail", + message: "no console", + file: "src/a.ts", + ...extra, +}); + +const noSource = (): undefined => undefined; + +describe("a fingerprint", () => { + it("is fp1 and 16 hex characters of the sha256 of its parts, joined by NUL", () => { + const parts = { + tool: "eslint", + rule: "no-console", + file: "src/a.ts", + anchor: "x", + occurrence: 1, + }; + expect(fingerprint(parts)).toBe("fp1:f60399d1c94096d8"); + expect(fingerprint({ ...parts, file: null })).toBe("fp1:ce4ebab2c81aa1e8"); + expect(fingerprint({ ...parts, occurrence: 2 })).not.toBe(fingerprint(parts)); + }); +}); + +describe("the order among repeats", () => { + const fp = (rule: string, file: string | null, anchor: string, occurrence: number): string => + fingerprint({ tool: "eslint", rule, file, anchor, occurrence }); + + it("counts each repeat by line, whatever order the tool listed them", () => { + const lines = [5, 2, 9, 2]; + const raws = lines.map((line) => found({ line })); + const done = completeFindings(raws, noSource); + const expected = [3, 1, 4, 2].map((n) => fp("no-console", "src/a.ts", "no console", n)); + expect(done.map((item) => item.fingerprint)).toEqual(expected); + }); + + it("counts repeats within one tool, rule, file and anchor, and no wider", () => { + const done = completeFindings( + [ + found({ rule: "a" }), + found({ rule: "b" }), + found({ rule: "a", file: "src/b.ts" }), + found({ rule: "a", message: "other" }), + found({ rule: "a" }), + ], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("a", "src/a.ts", "no console", 1), + fp("b", "src/a.ts", "no console", 1), + fp("a", "src/b.ts", "no console", 1), + fp("a", "src/a.ts", "other", 1), + fp("a", "src/a.ts", "no console", 2), + ]); + }); + + it("keeps the parts of the key apart: a rule and a file that read the same joined are not repeats", () => { + const done = completeFindings( + [found({ rule: "ab", file: "c" }), found({ rule: "a", file: "bc" })], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("ab", "c", "no console", 1), + fp("a", "bc", "no console", 1), + ]); + }); + + it("treats no file as an empty one, and a file named like the placeholder as its own", () => { + const none: RawFinding = { tool: "eslint", rule: "r", verdict: "fail", message: "m" }; + const done = completeFindings( + [ + none, + found({ rule: "r", file: "", message: "m" }), + found({ rule: "r", file: "x", message: "m" }), + ], + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual([ + fp("r", null, "m", 1), + fp("r", "", "m", 2), + fp("r", "x", "m", 1), + ]); + }); +}); + +describe("what a completed finding carries beyond the shape", () => { + it("has required, ack and survivor only when the finding had them", () => { + const plain = completeFindings([found()], noSource)[0]; + expect(Object.keys(plain ?? {}).sort()).toEqual([ + "file", + "fingerprint", + "law", + "line", + "message", + "rule", + "tool", + "verdict", + ]); + const survivor = { + file: "src/a.ts", + function: "f", + mutator: "m", + original: "o", + replacement: "r", + occurrence: 1, + }; + const full = completeFindings( + [found({ required: false, survivor: { ...survivor }, ack: undefined })], + noSource, + )[0]; + expect(full).toMatchObject({ required: false, survivor }); + expect("ack" in (full ?? {})).toBe(false); + }); +}); diff --git a/src/report/findings.ts b/src/report/findings.ts index e712eb7..1b67ed1 100644 --- a/src/report/findings.ts +++ b/src/report/findings.ts @@ -65,11 +65,17 @@ function byLine(raws: readonly RawFinding[]): (left: number, right: number) => n return (left, right) => lineAt(raws, left) - lineAt(raws, right) || left - right; } +interface Anchored { + readonly raw: RawFinding; + readonly anchor: string; +} + /** Each finding's order among its repeats, by line and then by the order the tool listed them. */ -function occurrences(raws: readonly RawFinding[], anchors: readonly string[]): number[] { +function occurrences(found: readonly Anchored[]): number[] { + const raws = found.map((item) => item.raw); const groups = new Map(); - raws.forEach((raw, index) => { - const key = groupKey(raw, anchors[index] ?? ""); + found.forEach(({ raw, anchor }, index) => { + const key = groupKey(raw, anchor); groups.set(key, [...(groups.get(key) ?? []), index]); }); const order = raws.map(() => 1); @@ -118,7 +124,7 @@ function complete(raw: RawFinding, anchor: string, occurrence: number): Finding * anchored on the flagged line (WEB_TOPS.md section 6.6). `read` returns a file's text, or undefined. */ export function completeFindings(raws: readonly RawFinding[], read: Reader): Finding[] { - const anchors = raws.map((raw) => anchorFor(raw, read)); - const order = occurrences(raws, anchors); - return raws.map((raw, index) => complete(raw, anchors[index] ?? "", order[index] ?? 1)); + const found = raws.map((raw) => ({ raw, anchor: anchorFor(raw, read) })); + const order = occurrences(found); + return found.map(({ raw, anchor }, index) => complete(raw, anchor, order[index] ?? 1)); } From aff845f7f37002840375da38c4408cc1274eb08b Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 05:32:20 -0500 Subject: [PATCH 3/8] Gate runner: keep the new edge tests out of existing test files, and make the diff coverage choice a function A changed test file brings every function it covers into the mutation scope, so the three cases added to the diff wiring, ratchet and tests-layer test files made the run exceed its budget. They now live in small files of their own that cover only what they pin, and the choice of coverage for a diff is a function with its own test, so the diff engine is not run for it. The diff engine option takes undefined, as its copy token option already does. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/scope.ts | 2 +- src/gate/diff-wiring.test.ts | 32 -------------- src/gate/ratchet-mode.test.ts | 64 +++++++++++++++++++++++++++ src/gate/ratchet.test.ts | 9 ---- src/gate/runner-edges.test.ts | 12 ++++- src/gate/runner.ts | 6 ++- src/layers/tests/launch-label.test.ts | 38 ++++++++++++++++ src/layers/tests/tests.test.ts | 11 ----- 8 files changed, 119 insertions(+), 55 deletions(-) create mode 100644 src/gate/ratchet-mode.test.ts create mode 100644 src/layers/tests/launch-label.test.ts diff --git a/src/diff/scope.ts b/src/diff/scope.ts index ec4f4f8..2daa6f8 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -24,7 +24,7 @@ export interface ScopeOptions { readonly base: string; readonly exec?: Exec; /** What a test file covers; by default a Vitest run of that file alone. */ - readonly coverageOf?: CoverageOf; + readonly coverageOf?: CoverageOf | undefined; /** Reasoned coverage exclusions from `web-tops.yaml`, passed to the default per-file run. */ readonly exclude?: readonly string[]; /** The random part of a merge-base copy's name; a test sets it to know the name. */ diff --git a/src/gate/diff-wiring.test.ts b/src/gate/diff-wiring.test.ts index 85f03e5..16111fb 100644 --- a/src/gate/diff-wiring.test.ts +++ b/src/gate/diff-wiring.test.ts @@ -130,38 +130,6 @@ describe("R63 the default CRAP scope is the diff, computed by the gate", () => { }); }); -describe("R63 a layer that does not need the tests' functions starts no test run", () => { - it("runs git and nothing else for a diff that changes a test file", async () => { - const dir = repoChanging(true); - write(dir, { "src/a.test.ts": "export const t = 1;\n" }); - commit(dir, "a test"); - const commands: string[] = []; - const exec: Exec = (command, args, options) => { - commands.push(command); - return realExec(command, args, options); - }; - const asking: Layer = { - ...fakeLayer({ id: "hygiene", gate: 1 }), - run: async (ctx: LayerContext) => { - await ctx.diff?.({ coveredByTests: false }); - return { verdict: "pass", findings: [], artifacts: {} }; - }, - }; - await runGate({ - layers: [asking], - config: cfg(), - options: opts({ base: "main" }), - cwd: dir, - env: {}, - exec, - outDir: join(dir, ".web-tops", "out"), - now: tickingClock(), - }); - expect(commands.length).toBeGreaterThan(0); - expect(new Set(commands)).toEqual(new Set(["git"])); - }); -}); - describe("R63 the provider computes the diff once for each kind of request", () => { /** How many git commands a run makes when its one layer asks for the diff as `asks` says. */ async function gitCalls(asks: readonly DiffNeeds[]): Promise { diff --git a/src/gate/ratchet-mode.test.ts b/src/gate/ratchet-mode.test.ts new file mode 100644 index 0000000..2de6098 --- /dev/null +++ b/src/gate/ratchet-mode.test.ts @@ -0,0 +1,64 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { canonicalJson } from "../baseline/canonical.ts"; +import type { Baseline } from "../baseline/types.ts"; +import { runGate } from "./runner.ts"; +import { cfg, fakeLayer, noExec, opts, tickingClock } from "./testkit.ts"; + +/** Only the ratchet mode reads the baseline a config names. */ + +const made: string[] = []; + +afterEach(() => { + for (const dir of made.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const BASELINE = ".web-tops/baseline.json"; + +const BASE: Baseline = { + report_version: "2.0.0", + subject: { package: "fixture", root: "." }, + crap: { "src/a.ts::f": { complexity: 3, coverage: 1, crap: 3 } }, + coverage: {}, + findings: {}, + fingerprints: [], + census: {}, + pages: {}, +}; + +function above(enforce: "hard" | "ratchet"): ReturnType { + const dir = mkdtempSync(join(tmpdir(), "wt-ratchet-mode-")); + made.push(dir); + mkdirSync(join(dir, ".web-tops")); + writeFileSync(join(dir, BASELINE), canonicalJson(BASE)); + const crap = fakeLayer({ + id: "crap", + gate: 4, + metrics: { + crap: [{ file: "src/a.ts", name: "f", line: 1, complexity: 3, coverage: 1, crap: 9 }], + }, + }); + return runGate({ + layers: [crap], + config: cfg({ enforce, baselinePath: BASELINE }), + options: opts(), + cwd: dir, + env: {}, + exec: noExec, + outDir: join(dir, ".web-tops", "out"), + now: tickingClock(), + }); +} + +describe("G9b: a run above its baseline", () => { + it("is left alone under hard, though the config names a baseline, and refused under ratchet", async () => { + const hard = await above("hard"); + expect(hard.layers[0]?.verdict).toBe("pass"); + expect(hard.layers[0]?.findings).toEqual([]); + expect(hard.passed).toBe(true); + const ratchet = await above("ratchet"); + expect(ratchet.layers[0]?.verdict).toBe("refuse"); + }); +}); diff --git a/src/gate/ratchet.test.ts b/src/gate/ratchet.test.ts index 5318558..2d9ca61 100644 --- a/src/gate/ratchet.test.ts +++ b/src/gate/ratchet.test.ts @@ -90,15 +90,6 @@ function layer(report: Report, id: string): Report["layers"][number] { return found; } -describe("G9b: only the ratchet mode reads a baseline", () => { - it("leaves a run above its baseline alone under hard, though the config names one", async () => { - const report = await run(project(BASE), layersWith(crapRows(9)), ratchet({ enforce: "hard" })); - expect(layer(report, "crap").verdict).toBe("pass"); - expect(layer(report, "crap").findings).toEqual([]); - expect(report.passed).toBe(true); - }); -}); - describe("G9b: a run above its baseline is refused, layer by layer", () => { it("passes a run that equals its baseline, and one that is below it", async () => { const same = await run( diff --git a/src/gate/runner-edges.test.ts b/src/gate/runner-edges.test.ts index b3ac960..7b346a7 100644 --- a/src/gate/runner-edges.test.ts +++ b/src/gate/runner-edges.test.ts @@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { Report } from "../report/types.ts"; -import { runGate, sourceReader } from "./runner.ts"; +import { coverageFor, runGate, sourceReader } from "./runner.ts"; import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; import type { GateConfig, GateOptions, Layer, LayerContext } from "./types.ts"; @@ -229,3 +229,13 @@ describe("the text of a file a finding points at", () => { expect(reads).toEqual(["/work:a.ts", "/work:b.ts"]); }); }); + +describe("the coverage the diff engine is given for a changed test file", () => { + it("is its own run for a layer that needs the tests' functions, and none for one that does not", async () => { + expect(coverageFor({ coveredByTests: true })).toBeUndefined(); + const none = coverageFor({ coveredByTests: false }); + const covered = await none?.(); + expect(covered).toBeInstanceOf(Map); + expect(covered?.size).toBe(0); + }); +}); diff --git a/src/gate/runner.ts b/src/gate/runner.ts index 9cb1ee1..62d6ec3 100644 --- a/src/gate/runner.ts +++ b/src/gate/runner.ts @@ -182,6 +182,10 @@ function keepArtifacts( /** The coverage a layer that does not need the tests' functions gets from the diff engine: none, without a Vitest run. */ const NO_COVERAGE: CoverageMap = new Map(); +/** What the diff engine is given to cover a changed test file with: none, for a layer that does not need the tests' functions, and its own run otherwise. */ +export const coverageFor = (needs: DiffNeeds): (() => Promise) | undefined => + needs.coveredByTests ? undefined : () => Promise.resolve(NO_COVERAGE); + /** The diff, computed the first time a layer asks and kept for each kind of ask (the test-covered functions cost Vitest runs). */ function diffProvider(input: RunInput, base: string): DiffProvider { const asked = new Map>(); @@ -196,7 +200,7 @@ function diffProvider(input: RunInput, base: string): DiffProvider { base, exec: input.exec, exclude, - ...(needs.coveredByTests ? {} : { coverageOf: () => Promise.resolve(NO_COVERAGE) }), + coverageOf: coverageFor(needs), }); asked.set(needs.coveredByTests, fresh); return fresh; diff --git a/src/layers/tests/launch-label.test.ts b/src/layers/tests/launch-label.test.ts new file mode 100644 index 0000000..280ac34 --- /dev/null +++ b/src/layers/tests/launch-label.test.ts @@ -0,0 +1,38 @@ +import { symlinkSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cfg } from "../../gate/testkit.ts"; +import type { Exec, LayerContext } from "../../gate/types.ts"; +import { cleanTemp, tempDir } from "../../laws/probe/testkit.ts"; +import { testsLayer } from "./index.ts"; + +afterAll(cleanTemp); + +const REPO = resolve(import.meta.dirname, "../../.."); + +function context(exec: Exec): LayerContext { + const cwd = tempDir(); + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + return { + cwd, + config: cfg(), + base: "origin/main", + enforce: "hard", + env: {}, + exec, + outDir: join(cwd, ".web-tops", "out"), + artifacts: new Map(), + }; +} + +describe("gate 2 says which tool it could not run", () => { + it("names vitest when it did not finish in time and was killed", async () => { + const exec: Exec = () => + Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-timeout" }); + const result = await testsLayer.run(context(exec)); + expect(result.verdict).toBe("fail"); + expect(result.findings).toMatchObject([ + { rule: "tool-timeout", message: "vitest did not finish in time and was killed" }, + ]); + }); +}); diff --git a/src/layers/tests/tests.test.ts b/src/layers/tests/tests.test.ts index fba5aa0..959602a 100644 --- a/src/layers/tests/tests.test.ts +++ b/src/layers/tests/tests.test.ts @@ -313,17 +313,6 @@ describe("G4a tests: zero tests, .only and .skip each fail", () => { expect(rulesOf(result)).toEqual([rule]); }); - it("names vitest in what it says when node is missing or vitest is killed", async () => { - const missing = await runTests(real({}), () => - Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-missing" as const }), - ); - expect(missing.findings[0]?.message).toBe("node is not on PATH, so vitest cannot run"); - const slow = await runTests(real({}), () => - Promise.resolve({ code: null, stdout: "", stderr: "", failure: "tool-timeout" as const }), - ); - expect(slow.findings[0]?.message).toBe("vitest did not finish in time and was killed"); - }); - it("results that are not JSON or not an object are findings, not passes", async () => { const item = real({}); const writes = From 8dc0c162619e00e69e0cdb946f70cf1c033b8803 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 05:51:16 -0500 Subject: [PATCH 4/8] Gate runner: choose the diff options in a function, and pin the leftover survivors The coverage choice for a diff is now the options a layer gets, in one tested function, so no line of the diff engine changes. The ratchet mode test only leaves a hard run alone, the line order of repeats has an ascending case, and the tool that is not installed has its own direct test. Co-Authored-By: Claude Sonnet 5.5 --- src/diff/scope.ts | 2 +- src/gate/ratchet-mode.test.ts | 6 ++---- src/gate/runner-edges.test.ts | 18 ++++++++++++------ src/gate/runner.ts | 16 +++++----------- src/layers/tests/launch-label.test.ts | 18 ++++++++++++++++-- src/report/findings-edges.test.ts | 10 ++++++++++ 6 files changed, 46 insertions(+), 24 deletions(-) diff --git a/src/diff/scope.ts b/src/diff/scope.ts index 2daa6f8..ec4f4f8 100644 --- a/src/diff/scope.ts +++ b/src/diff/scope.ts @@ -24,7 +24,7 @@ export interface ScopeOptions { readonly base: string; readonly exec?: Exec; /** What a test file covers; by default a Vitest run of that file alone. */ - readonly coverageOf?: CoverageOf | undefined; + readonly coverageOf?: CoverageOf; /** Reasoned coverage exclusions from `web-tops.yaml`, passed to the default per-file run. */ readonly exclude?: readonly string[]; /** The random part of a merge-base copy's name; a test sets it to know the name. */ diff --git a/src/gate/ratchet-mode.test.ts b/src/gate/ratchet-mode.test.ts index 2de6098..b16c0b8 100644 --- a/src/gate/ratchet-mode.test.ts +++ b/src/gate/ratchet-mode.test.ts @@ -28,7 +28,7 @@ const BASE: Baseline = { pages: {}, }; -function above(enforce: "hard" | "ratchet"): ReturnType { +function above(enforce: "hard"): ReturnType { const dir = mkdtempSync(join(tmpdir(), "wt-ratchet-mode-")); made.push(dir); mkdirSync(join(dir, ".web-tops")); @@ -53,12 +53,10 @@ function above(enforce: "hard" | "ratchet"): ReturnType { } describe("G9b: a run above its baseline", () => { - it("is left alone under hard, though the config names a baseline, and refused under ratchet", async () => { + it("is left alone under hard, though the config names a baseline", async () => { const hard = await above("hard"); expect(hard.layers[0]?.verdict).toBe("pass"); expect(hard.layers[0]?.findings).toEqual([]); expect(hard.passed).toBe(true); - const ratchet = await above("ratchet"); - expect(ratchet.layers[0]?.verdict).toBe("refuse"); }); }); diff --git a/src/gate/runner-edges.test.ts b/src/gate/runner-edges.test.ts index 7b346a7..33a57dc 100644 --- a/src/gate/runner-edges.test.ts +++ b/src/gate/runner-edges.test.ts @@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { Report } from "../report/types.ts"; -import { coverageFor, runGate, sourceReader } from "./runner.ts"; +import { optionsFor, runGate, sourceReader } from "./runner.ts"; import { cfg, fakeLayer, noExec, off, opts, tempProject, tickingClock } from "./testkit.ts"; import type { GateConfig, GateOptions, Layer, LayerContext } from "./types.ts"; @@ -230,11 +230,17 @@ describe("the text of a file a finding points at", () => { }); }); -describe("the coverage the diff engine is given for a changed test file", () => { - it("is its own run for a layer that needs the tests' functions, and none for one that does not", async () => { - expect(coverageFor({ coveredByTests: true })).toBeUndefined(); - const none = coverageFor({ coveredByTests: false }); - const covered = await none?.(); +describe("the diff engine's options for a layer", () => { + const base = { cwd: "/work", base: "main", exclude: ["src/gen/**"] }; + + it("are kept for a layer that needs the tests' functions", () => { + expect(optionsFor({ coveredByTests: true }, base)).toBe(base); + }); + + it("add a coverage of none for one that does not, and keep the rest", async () => { + const given = optionsFor({ coveredByTests: false }, base); + expect(given).toMatchObject(base); + const covered = await given.coverageOf?.("src/a.test.ts"); expect(covered).toBeInstanceOf(Map); expect(covered?.size).toBe(0); }); diff --git a/src/gate/runner.ts b/src/gate/runner.ts index 62d6ec3..eab775b 100644 --- a/src/gate/runner.ts +++ b/src/gate/runner.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join, relative, resolve, sep } from "node:path"; import { applyRatchet } from "../baseline/ratchet.ts"; -import { diffScope, type CoverageMap, type ScopeResult } from "../diff/index.ts"; +import { diffScope, type CoverageMap, type ScopeOptions, type ScopeResult } from "../diff/index.ts"; import { COUNTED_LAYERS, completeFindings, countsByRuleFile } from "../report/findings.ts"; import { endsRun, @@ -182,9 +182,9 @@ function keepArtifacts( /** The coverage a layer that does not need the tests' functions gets from the diff engine: none, without a Vitest run. */ const NO_COVERAGE: CoverageMap = new Map(); -/** What the diff engine is given to cover a changed test file with: none, for a layer that does not need the tests' functions, and its own run otherwise. */ -export const coverageFor = (needs: DiffNeeds): (() => Promise) | undefined => - needs.coveredByTests ? undefined : () => Promise.resolve(NO_COVERAGE); +/** The diff engine's options for a layer: a layer that does not need the tests' functions covers a changed test file with none, and the engine's own run is kept for one that does. */ +export const optionsFor = (needs: DiffNeeds, options: ScopeOptions): ScopeOptions => + needs.coveredByTests ? options : { ...options, coverageOf: () => Promise.resolve(NO_COVERAGE) }; /** The diff, computed the first time a layer asks and kept for each kind of ask (the test-covered functions cost Vitest runs). */ function diffProvider(input: RunInput, base: string): DiffProvider { @@ -195,13 +195,7 @@ function diffProvider(input: RunInput, base: string): DiffProvider { return known; } const exclude = input.config.coverage.exclude.map((entry) => entry.glob); - const fresh = diffScope({ - cwd: input.cwd, - base, - exec: input.exec, - exclude, - coverageOf: coverageFor(needs), - }); + const fresh = diffScope(optionsFor(needs, { cwd: input.cwd, base, exec: input.exec, exclude })); asked.set(needs.coveredByTests, fresh); return fresh; }; diff --git a/src/layers/tests/launch-label.test.ts b/src/layers/tests/launch-label.test.ts index 280ac34..a0b42e8 100644 --- a/src/layers/tests/launch-label.test.ts +++ b/src/layers/tests/launch-label.test.ts @@ -4,15 +4,18 @@ import { afterAll, describe, expect, it } from "vitest"; import { cfg } from "../../gate/testkit.ts"; import type { Exec, LayerContext } from "../../gate/types.ts"; import { cleanTemp, tempDir } from "../../laws/probe/testkit.ts"; +import { runTool } from "../hygiene/tooling.ts"; import { testsLayer } from "./index.ts"; afterAll(cleanTemp); const REPO = resolve(import.meta.dirname, "../../.."); -function context(exec: Exec): LayerContext { +function context(exec: Exec, installed = true): LayerContext { const cwd = tempDir(); - symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + if (installed) { + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + } return { cwd, config: cfg(), @@ -35,4 +38,15 @@ describe("gate 2 says which tool it could not run", () => { { rule: "tool-timeout", message: "vitest did not finish in time and was killed" }, ]); }); + + it("is tool-missing, with nothing on stdout, when vitest is not installed in the project", async () => { + const exec: Exec = () => Promise.reject(new Error("nothing is started")); + const result = await runTool(context(exec, false), "vitest", []); + expect(result).toEqual({ + code: null, + stdout: "", + stderr: "vitest is not installed in node_modules (package vitest)", + failure: "tool-missing", + }); + }); }); diff --git a/src/report/findings-edges.test.ts b/src/report/findings-edges.test.ts index fe694b3..81cd56f 100644 --- a/src/report/findings-edges.test.ts +++ b/src/report/findings-edges.test.ts @@ -41,6 +41,16 @@ describe("the order among repeats", () => { expect(done.map((item) => item.fingerprint)).toEqual(expected); }); + it("counts repeats already in line order as they are", () => { + const done = completeFindings( + [1, 2, 3].map((line) => found({ line })), + noSource, + ); + expect(done.map((item) => item.fingerprint)).toEqual( + [1, 2, 3].map((n) => fp("no-console", "src/a.ts", "no console", n)), + ); + }); + it("counts repeats within one tool, rule, file and anchor, and no wider", () => { const done = completeFindings( [ From 38cd55dda650ca4aab88e7208baadbd8aac4d663 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 05:52:59 -0500 Subject: [PATCH 5/8] Laws layer: run it end to end on a real project, and say why a config that reaches no TypeScript file cannot be judged A project whose ESLint config has no block for TypeScript files gives the reader no configuration for them, so W-01 was not-evaluated with the reason that no TypeScript file was linted. That is the right verdict, and the reason now says why: the config reaches no TypeScript file, so no rule can be judged. The end to end test runs the layer in a child process over a real project with its own ESLint and TypeScript: W-01 and W-04 pass on the strict type-checked preset with every flag set, W-01 is refused at the config line of an override that sets no-explicit-any to off and of one that sets it to warn, and the config with no TypeScript block is not-evaluated with the new reason and its file refused. Co-Authored-By: Claude Sonnet 5.5 --- src/laws/layer/real-e2e.test.ts | 169 ++++++++++++++++++++++++++ src/laws/readers/eslint-judge.test.ts | 3 +- src/laws/readers/eslint-judge.ts | 7 +- 3 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 src/laws/layer/real-e2e.test.ts diff --git a/src/laws/layer/real-e2e.test.ts b/src/laws/layer/real-e2e.test.ts new file mode 100644 index 0000000..9d98547 --- /dev/null +++ b/src/laws/layer/real-e2e.test.ts @@ -0,0 +1,169 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cleanTemp, tempDir } from "../probe/testkit.ts"; + +/** + * The laws layer on a real project, with the project's own ESLint and TypeScript. The layer runs in a child process, so + * what is covered here is the process boundary and the tools, not the layer's code, which has its own tests. + */ + +afterAll(cleanTemp); + +const REPO = resolve(import.meta.dirname, "../../.."); +const ANY = "@typescript-eslint/no-explicit-any"; +const SLOW = 120_000; + +const TSCONFIG = JSON.stringify({ + compilerOptions: { + target: "es2022", + module: "nodenext", + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + noImplicitOverride: true, + noFallthroughCasesInSwitch: true, + verbatimModuleSyntax: true, + isolatedModules: true, + erasableSyntaxOnly: true, + }, + include: ["src"], +}); + +/** The config of a project that lints its TypeScript with the strict type-checked preset; `extra` is one more block. */ +function config(extra: string): string { + return [ + 'import tseslint from "typescript-eslint";', + "export default tseslint.config(", + ' { files: ["**/*.ts"], extends: [tseslint.configs.strictTypeChecked], languageOptions: { parserOptions: { projectService: true } } },', + extra, + ");", + "", + ].join("\n"); +} + +/** The script the child process runs: the real layer with the real exec and readers, over W-01 and W-04 of the catalog. */ +function layerScript(): string { + return [ + `const root = process.env.WT_REPO;`, + `const { realExec } = await import(root + "/src/gate/exec.ts");`, + `const { lawsLayer } = await import(root + "/src/laws/layer/index.ts");`, + `const { yamlCatalog } = await import(root + "/src/laws/layer/catalog.ts");`, + `const { cfg } = await import(root + "/src/gate/testkit.ts");`, + `const cwd = process.argv[1];`, + `const laws = yamlCatalog().filter((law) => law.id === "W-01" || law.id === "W-04");`, + `const result = await lawsLayer(() => laws).run({`, + ` cwd, config: cfg({ class: "ts-library" }), base: "origin/main", enforce: "hard",`, + ` env: { PATH: process.env.PATH ?? "" }, exec: realExec, outDir: cwd + "/.web-tops/out",`, + ` artifacts: new Map([["eslint-json", cwd + "/out/e.json"], ["tsc-diagnostics", cwd + "/out/t.json"]]),`, + `});`, + `process.stdout.write(JSON.stringify({ verdict: result.verdict, findings: result.findings }));`, + ].join("\n"); +} + +interface Answer { + readonly verdict: string; + readonly findings: readonly { + readonly rule: string; + readonly verdict: string; + readonly file?: string; + readonly line?: number; + readonly message: string; + }[]; +} + +function isAnswer(value: unknown): value is Answer { + return typeof value === "object" && value !== null && "verdict" in value && "findings" in value; +} + +function git(cwd: string, ...args: string[]): void { + const done = spawnSync("git", args, { cwd, encoding: "utf8" }); + if (done.status !== 0) { + throw new Error(`git ${args.join(" ")}: ${done.stderr}`); + } +} + +function project(eslintConfig: string): string { + const cwd = tempDir(); + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + mkdirSync(join(cwd, "src")); + mkdirSync(join(cwd, "out")); + writeFileSync(join(cwd, "package.json"), '{ "type": "module" }\n'); + writeFileSync(join(cwd, "tsconfig.json"), TSCONFIG); + writeFileSync(join(cwd, "src/a.ts"), "export const a: number = 1;\n"); + writeFileSync(join(cwd, "eslint.config.js"), eslintConfig); + writeFileSync(join(cwd, "out/e.json"), "[]\n"); + writeFileSync(join(cwd, "out/t.json"), "[]\n"); + git(cwd, "init", "-q"); + git(cwd, "add", "-A"); + return cwd; +} + +function judge(eslintConfig: string): Answer { + const cwd = project(eslintConfig); + const done = spawnSync("bun", ["-e", layerScript(), cwd], { + cwd, + env: { ...process.env, WT_REPO: REPO }, + encoding: "utf8", + }); + if (done.status !== 0) { + throw new Error(`the layer did not run: ${done.stderr}`); + } + const parsed: unknown = JSON.parse(done.stdout); + if (!isAnswer(parsed)) { + throw new Error(`the layer answered something else: ${done.stdout}`); + } + return parsed; +} + +const lawOf = (answer: Answer, id: string): Answer["findings"][number] | undefined => + answer.findings.find((found) => found.rule === `law/${id}`); + +describe("G4 the laws layer on a project with real ESLint and real tsc", () => { + it( + "passes W-01 and W-04 when every rule is at error for every TypeScript file and every flag is set", + () => { + const answer = judge(config("")); + expect(lawOf(answer, "W-01")?.verdict).toBe("pass"); + expect(lawOf(answer, "W-04")?.verdict).toBe("pass"); + expect(answer.verdict).toBe("pass"); + }, + SLOW, + ); + + it.each([ + ["off", "off"], + ["warn", "warn"], + ])( + "refuses W-01 at the config line of an override that sets no-explicit-any to %s", + (_name, level) => { + const override = ` { files: ["src/**"], rules: { "${ANY}": "${level}" } },`; + const text = config(override); + const line = text.split("\n").indexOf(override) + 1; + const answer = judge(text); + expect(lawOf(answer, "W-01")?.verdict).toBe("refuse"); + expect(answer.verdict).toBe("refuse"); + const spot = answer.findings.find((found) => found.rule === ANY); + expect(spot).toMatchObject({ verdict: "refuse", file: "eslint.config.js", line }); + expect(spot?.message).toBe(`${ANY} is below error for 1 file(s), such as src/a.ts`); + expect(lawOf(answer, "W-04")?.verdict).toBe("pass"); + }, + SLOW, + ); + + it( + "says why W-01 is not evaluated when the config reaches no TypeScript file, and refuses the file-set", + () => { + const answer = judge(`export default [{ rules: { "${ANY}": "error" } }];\n`); + const reading = answer.findings.find((found) => found.rule === "law/W-01"); + expect(reading?.verdict).toBe("not-evaluated"); + expect(reading?.message).toContain("ESLint's config reaches no TypeScript file"); + expect(answer.findings.find((found) => found.rule === "file-set")).toMatchObject({ + verdict: "refuse", + file: "src/a.ts", + }); + }, + SLOW, + ); +}); diff --git a/src/laws/readers/eslint-judge.test.ts b/src/laws/readers/eslint-judge.test.ts index 5788ed7..2050fcc 100644 --- a/src/laws/readers/eslint-judge.test.ts +++ b/src/laws/readers/eslint-judge.test.ts @@ -153,7 +153,8 @@ describe("G4a the ESLint reader judges a law by each file's effective config", ( expect(reading).toEqual({ law: "W-01", verdict: "not-evaluated", - reason: "no TypeScript file was linted", + reason: + "no TypeScript file was linted: ESLint's config reaches no TypeScript file (no config block has files for **/*.ts and the TypeScript parser, or an ignore drops them), so no rule can be judged", findings: [], }); }); diff --git a/src/laws/readers/eslint-judge.ts b/src/laws/readers/eslint-judge.ts index bcf5f57..9d3d8a9 100644 --- a/src/laws/readers/eslint-judge.ts +++ b/src/laws/readers/eslint-judge.ts @@ -160,11 +160,16 @@ function countedReading( : undefined; } +/** Why no rule can be judged, worded for an adopter whose ESLint config cannot reach `.ts` files at all. */ +function unlintedReason(): string { + return "no TypeScript file was linted: ESLint's config reaches no TypeScript file (no config block has files for **/*.ts and the TypeScript parser, or an ignore drops them), so no rule can be judged"; +} + /** No TypeScript file was linted: the typed rules have no file to be judged over. */ function unjudgedReading(law: string, evidence: Evidence): LawReading | undefined { return [...evidence.levels.keys()].some(isTypeScript) ? undefined - : { law, verdict: "not-evaluated", reason: "no TypeScript file was linted", findings: [] }; + : { law, verdict: "not-evaluated", reason: unlintedReason(), findings: [] }; } function passReading(law: string, counted: readonly string[]): LawReading { From 080b88c4c43150d61744e6494e5ce1b1c43e73ce Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 06:05:04 -0500 Subject: [PATCH 6/8] Gate runner: leave ties to the stable sort, and write the test baseline without the canonical writer Repeats on the same line stay in the order the tool listed them because the sort is stable, so the index tie break said nothing the sort did not. The ratchet mode test writes its baseline as plain JSON so it covers no code of the baseline module. Co-Authored-By: Claude Sonnet 5.5 --- src/gate/ratchet-mode.test.ts | 3 +-- src/report/findings.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/gate/ratchet-mode.test.ts b/src/gate/ratchet-mode.test.ts index b16c0b8..5dbd2ec 100644 --- a/src/gate/ratchet-mode.test.ts +++ b/src/gate/ratchet-mode.test.ts @@ -2,7 +2,6 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { canonicalJson } from "../baseline/canonical.ts"; import type { Baseline } from "../baseline/types.ts"; import { runGate } from "./runner.ts"; import { cfg, fakeLayer, noExec, opts, tickingClock } from "./testkit.ts"; @@ -32,7 +31,7 @@ function above(enforce: "hard"): ReturnType { const dir = mkdtempSync(join(tmpdir(), "wt-ratchet-mode-")); made.push(dir); mkdirSync(join(dir, ".web-tops")); - writeFileSync(join(dir, BASELINE), canonicalJson(BASE)); + writeFileSync(join(dir, BASELINE), JSON.stringify(BASE)); const crap = fakeLayer({ id: "crap", gate: 4, diff --git a/src/report/findings.ts b/src/report/findings.ts index 1b67ed1..34aa8fb 100644 --- a/src/report/findings.ts +++ b/src/report/findings.ts @@ -62,7 +62,7 @@ function lineAt(raws: readonly RawFinding[], index: number): number { } function byLine(raws: readonly RawFinding[]): (left: number, right: number) => number { - return (left, right) => lineAt(raws, left) - lineAt(raws, right) || left - right; + return (left, right) => lineAt(raws, left) - lineAt(raws, right); } interface Anchored { From 93347a635f9c45d029f26a5fb483664bdb0c281a Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 07:30:18 -0500 Subject: [PATCH 7/8] Tests: run the ratchet end to end test through the CLI as a process, so it covers no in-process code The end to end test ran the whole gate in this process: 38 s of real git and real tools that covered 86 percent of the runner, so every mutant in the runner, the plan and the findings also ran it. It now starts bun src/cli/main.ts for check, gate and baseline, with the same assertions on exit codes, output and report, so it is still the check of the entry point and of the ratchet adoption, and the mutants of the code it ran are judged by the tests that stand beside that code. Co-Authored-By: Claude Sonnet 5.5 --- src/cli/ratchet-e2e.test.ts | 86 ++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/src/cli/ratchet-e2e.test.ts b/src/cli/ratchet-e2e.test.ts index bbab443..14fa37b 100644 --- a/src/cli/ratchet-e2e.test.ts +++ b/src/cli/ratchet-e2e.test.ts @@ -1,15 +1,12 @@ +import { spawn } from "node:child_process"; import { existsSync, readFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { validateReport, type GateReport } from "../config/report.ts"; import { cleanRepos, commit, git, write } from "../diff/test-support.ts"; import { REAL_TOOL_TIMEOUT } from "../gate/test-support-timeouts.ts"; import { realProject } from "../layers/tests/testkit.ts"; import { readBaseline } from "../baseline/read.ts"; -import { runBaselineCli } from "./baseline.ts"; -import { runCheckCli } from "./check.ts"; -import { runGateCli } from "./gate.ts"; -import type { Io } from "./io.ts"; /** * G9 end to end: a project with legacy debt (a function over the cap that no test runs, and coverage under the floor) @@ -81,31 +78,41 @@ function adopter(yaml = YAML, files: Readonly> = DEBT_FIL return project.dir; } -function streams(cwd: string): { io: Io; out: () => string; err: () => string } { - let out = ""; - let err = ""; - return { - io: { - out: (text) => { - out += text; - }, - err: (text) => { - err += text; - }, - cwd, - env: { ...process.env, CI: "" }, - }, - out: () => out, - err: () => err, - }; +const MAIN = resolve(import.meta.dirname, "main.ts"); + +interface Ran { + readonly code: number; + readonly out: string; + readonly err: string; +} + +/** + * The real CLI as a process: the gate it runs is not this test's own, so the tests that stand beside the code + * are the ones that judge it, and this one stays an end to end check of the entry point. + */ +function cli(cwd: string, ...args: string[]): Promise { + return new Promise((done, fail) => { + const child = spawn("bun", [MAIN, ...args], { cwd, env: { ...process.env, CI: "" } }); + let out = ""; + let err = ""; + child.stdout.on("data", (chunk: Buffer) => { + out += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + err += chunk.toString(); + }); + child.on("error", fail); + child.on("close", (code) => { + done({ code: code ?? 1, out, err }); + }); + }); } async function gate(dir: string, ...args: string[]): Promise<{ code: number; report: GateReport }> { - const run = streams(dir); - const code = await runGateCli(args, run.io); + const run = await cli(dir, "gate", ...args); const read = validateReport(JSON.parse(readFileSync(join(dir, ".web-tops/report.json"), "utf8"))); if (!read.ok) throw new Error(`the report is not valid: ${JSON.stringify(read.problems)}`); - return { code, report: read.value }; + return { code: run.code, report: read.value }; } function layerOf(report: GateReport, layer: string): GateReport["layers"][number] | undefined { @@ -121,11 +128,10 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () "takes its first baseline from a failing run, then passes, refuses new debt, and does not refuse legacy it touches", async () => { const dir = adopter(); - const checked = streams(dir); - const code = await runCheckCli([], checked.io); - expect(checked.err()).toBe(""); - expect(code).toBe(0); - expect(checked.out()).toContain( + const checked = await cli(dir, "check"); + expect(checked.err).toBe(""); + expect(checked.code).toBe(0); + expect(checked.out).toContain( "no baseline yet: run webtops gate --all --keep-going, then webtops baseline", ); @@ -136,9 +142,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () expect(rulesOf(first.report, "coverage")).toContain("coverage-below-floor"); // 2. a baseline is taken from it: the legacy findings stay in it - const taken = streams(dir); - expect(await runBaselineCli([], taken.io)).toBe(0); - expect(taken.out()).toMatch(/^wrote \.web-tops\/baseline\.json: 2 functions, 1 packages/); + const taken = await cli(dir, "baseline"); + expect(taken.code).toBe(0); + expect(taken.out).toMatch(/^wrote \.web-tops\/baseline\.json: 2 functions, 1 packages/); commit(dir, "adopt the ratchet"); // 3. held to it, with nothing above it, the run passes: no placeholder stands in, and the legacy debt is not judged anew @@ -172,9 +178,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () async () => { const dir = adopter(); await gate(dir, "--keep-going"); - const run = streams(dir); - expect(await runBaselineCli([], run.io)).toBe(2); - expect(run.err()).toContain("--all"); + const run = await cli(dir, "baseline"); + expect(run.code).toBe(2); + expect(run.err).toContain("--all"); expect(existsSync(join(dir, ".web-tops/baseline.json"))).toBe(false); }, REAL_TOOL_TIMEOUT * 3, @@ -188,9 +194,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () commit(dir, "break a test"); const first = await gate(dir, "--all", "--keep-going"); expect(first.code).toBe(1); - const run = streams(dir); - expect(await runBaselineCli([], run.io)).toBe(2); - expect(run.err()).toContain("what failed is not something a first baseline holds"); + const run = await cli(dir, "baseline"); + expect(run.code).toBe(2); + expect(run.err).toContain("what failed is not something a first baseline holds"); expect(existsSync(join(dir, ".web-tops/baseline.json"))).toBe(false); }, REAL_TOOL_TIMEOUT * 3, @@ -238,7 +244,7 @@ describe("G9-4 end to end: legacy lint adopts the ratchet, and is held at its co const first = await gate(dir, "--all", "--keep-going"); expect(first.code).toBe(1); expect(rulesOf(first.report, "hygiene")).toContain("no-var"); - expect(await runBaselineCli([], streams(dir).io)).toBe(0); + expect((await cli(dir, "baseline")).code).toBe(0); commit(dir, "adopt the ratchet"); const read = readBaseline(dir, ".web-tops/baseline.json"); expect(read.ok && read.baseline.findings).toEqual({ "no-var": { "src/old.ts": 2 } }); From e05a7dd73f0ae544458525b121778207118c8a43 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 08:27:19 -0500 Subject: [PATCH 8/8] Revert: run the ratchet end to end test through the CLI as a process A changed test file brings the functions of its merge base copy into the mutation scope, and the in-process version of this test covered the whole gate, so editing it put every layer, the baseline code and the report code in scope: 2757 mutants and 248 survivors of old code instead of 570 and none. The test is back as it was. Co-Authored-By: Claude Sonnet 5.5 --- src/cli/ratchet-e2e.test.ts | 86 +++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/src/cli/ratchet-e2e.test.ts b/src/cli/ratchet-e2e.test.ts index 14fa37b..bbab443 100644 --- a/src/cli/ratchet-e2e.test.ts +++ b/src/cli/ratchet-e2e.test.ts @@ -1,12 +1,15 @@ -import { spawn } from "node:child_process"; import { existsSync, readFileSync, rmSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { validateReport, type GateReport } from "../config/report.ts"; import { cleanRepos, commit, git, write } from "../diff/test-support.ts"; import { REAL_TOOL_TIMEOUT } from "../gate/test-support-timeouts.ts"; import { realProject } from "../layers/tests/testkit.ts"; import { readBaseline } from "../baseline/read.ts"; +import { runBaselineCli } from "./baseline.ts"; +import { runCheckCli } from "./check.ts"; +import { runGateCli } from "./gate.ts"; +import type { Io } from "./io.ts"; /** * G9 end to end: a project with legacy debt (a function over the cap that no test runs, and coverage under the floor) @@ -78,41 +81,31 @@ function adopter(yaml = YAML, files: Readonly> = DEBT_FIL return project.dir; } -const MAIN = resolve(import.meta.dirname, "main.ts"); - -interface Ran { - readonly code: number; - readonly out: string; - readonly err: string; -} - -/** - * The real CLI as a process: the gate it runs is not this test's own, so the tests that stand beside the code - * are the ones that judge it, and this one stays an end to end check of the entry point. - */ -function cli(cwd: string, ...args: string[]): Promise { - return new Promise((done, fail) => { - const child = spawn("bun", [MAIN, ...args], { cwd, env: { ...process.env, CI: "" } }); - let out = ""; - let err = ""; - child.stdout.on("data", (chunk: Buffer) => { - out += chunk.toString(); - }); - child.stderr.on("data", (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on("error", fail); - child.on("close", (code) => { - done({ code: code ?? 1, out, err }); - }); - }); +function streams(cwd: string): { io: Io; out: () => string; err: () => string } { + let out = ""; + let err = ""; + return { + io: { + out: (text) => { + out += text; + }, + err: (text) => { + err += text; + }, + cwd, + env: { ...process.env, CI: "" }, + }, + out: () => out, + err: () => err, + }; } async function gate(dir: string, ...args: string[]): Promise<{ code: number; report: GateReport }> { - const run = await cli(dir, "gate", ...args); + const run = streams(dir); + const code = await runGateCli(args, run.io); const read = validateReport(JSON.parse(readFileSync(join(dir, ".web-tops/report.json"), "utf8"))); if (!read.ok) throw new Error(`the report is not valid: ${JSON.stringify(read.problems)}`); - return { code: run.code, report: read.value }; + return { code, report: read.value }; } function layerOf(report: GateReport, layer: string): GateReport["layers"][number] | undefined { @@ -128,10 +121,11 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () "takes its first baseline from a failing run, then passes, refuses new debt, and does not refuse legacy it touches", async () => { const dir = adopter(); - const checked = await cli(dir, "check"); - expect(checked.err).toBe(""); - expect(checked.code).toBe(0); - expect(checked.out).toContain( + const checked = streams(dir); + const code = await runCheckCli([], checked.io); + expect(checked.err()).toBe(""); + expect(code).toBe(0); + expect(checked.out()).toContain( "no baseline yet: run webtops gate --all --keep-going, then webtops baseline", ); @@ -142,9 +136,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () expect(rulesOf(first.report, "coverage")).toContain("coverage-below-floor"); // 2. a baseline is taken from it: the legacy findings stay in it - const taken = await cli(dir, "baseline"); - expect(taken.code).toBe(0); - expect(taken.out).toMatch(/^wrote \.web-tops\/baseline\.json: 2 functions, 1 packages/); + const taken = streams(dir); + expect(await runBaselineCli([], taken.io)).toBe(0); + expect(taken.out()).toMatch(/^wrote \.web-tops\/baseline\.json: 2 functions, 1 packages/); commit(dir, "adopt the ratchet"); // 3. held to it, with nothing above it, the run passes: no placeholder stands in, and the legacy debt is not judged anew @@ -178,9 +172,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () async () => { const dir = adopter(); await gate(dir, "--keep-going"); - const run = await cli(dir, "baseline"); - expect(run.code).toBe(2); - expect(run.err).toContain("--all"); + const run = streams(dir); + expect(await runBaselineCli([], run.io)).toBe(2); + expect(run.err()).toContain("--all"); expect(existsSync(join(dir, ".web-tops/baseline.json"))).toBe(false); }, REAL_TOOL_TIMEOUT * 3, @@ -194,9 +188,9 @@ describe("G9 end to end: legacy debt adopts the ratchet, and is held to it", () commit(dir, "break a test"); const first = await gate(dir, "--all", "--keep-going"); expect(first.code).toBe(1); - const run = await cli(dir, "baseline"); - expect(run.code).toBe(2); - expect(run.err).toContain("what failed is not something a first baseline holds"); + const run = streams(dir); + expect(await runBaselineCli([], run.io)).toBe(2); + expect(run.err()).toContain("what failed is not something a first baseline holds"); expect(existsSync(join(dir, ".web-tops/baseline.json"))).toBe(false); }, REAL_TOOL_TIMEOUT * 3, @@ -244,7 +238,7 @@ describe("G9-4 end to end: legacy lint adopts the ratchet, and is held at its co const first = await gate(dir, "--all", "--keep-going"); expect(first.code).toBe(1); expect(rulesOf(first.report, "hygiene")).toContain("no-var"); - expect((await cli(dir, "baseline")).code).toBe(0); + expect(await runBaselineCli([], streams(dir).io)).toBe(0); commit(dir, "adopt the ratchet"); const read = readBaseline(dir, ".web-tops/baseline.json"); expect(read.ok && read.baseline.findings).toEqual({ "no-var": { "src/old.ts": 2 } });