diff --git a/src/gate/types.ts b/src/gate/types.ts index 8567058..74fdaee 100644 --- a/src/gate/types.ts +++ b/src/gate/types.ts @@ -91,6 +91,12 @@ export interface LayerContext { export interface Consumed { readonly artifact: string; readonly from: string; + /** + * The runner passes the artifact if the producer left it and skips it if not, where a missing input otherwise makes + * the layer `not-run`. The layer says what a missing one means for what it judges (R92): a required law whose + * artifact is missing is `not-evaluated`, which ends the run. + */ + readonly optional?: boolean; } export interface Layer { diff --git a/src/laws/layer/catalog.test.ts b/src/laws/layer/catalog.test.ts new file mode 100644 index 0000000..51de24e --- /dev/null +++ b/src/laws/layer/catalog.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import type { Catalog, Law } from "../catalog/index.ts"; +import { entryOf, yamlCatalog } from "./catalog.ts"; + +const law = (id: string, extra: Partial = {}): Law => ({ + id, + title: "t", + family: "Types", + severity: "R", + classes: ["ts-library", "node-service"], + reader: { kinds: ["tl"], text: "tl: x" }, + rules: [{ tool: "eslint", name: "no-explicit-any" }], + judged: "v0.1", + rationale: "r", + fix: "f", + ...extra, +}); + +describe("G4a the laws layer takes its laws from the catalog's files", () => { + it("a law is its id, severity, first reader kind, classes and rules", () => { + expect(entryOf(law("W-01"))).toEqual({ + id: "W-01", + severity: "R", + reader: "tl", + classes: ["ts-library", "node-service"], + judged: true, + rules: [{ tool: "eslint", name: "no-explicit-any" }], + }); + expect( + entryOf(law("W-03", { severity: "A", reader: { kinds: ["at", "an"], text: "" } })), + ).toMatchObject({ + severity: "A", + reader: "at", + }); + }); + + it("a law with no reader kind is not built yet", () => { + expect(entryOf(law("W-03", { reader: { kinds: [], text: "" } })).reader).toBe("tbd"); + }); + + it("a law is judged when the catalog says v0.1 or that M3 gives it a reader, and not when it says no", () => { + expect( + ["v0.1", "m3", "no"].map( + (judged) => + entryOf(law("W-09", { judged: judged === "no" ? "no" : judged === "m3" ? "m3" : "v0.1" })) + .judged, + ), + ).toEqual([true, true, false]); + }); +}); + +describe("G4a the catalog of the kit", () => { + it("maps every law the loader gives, in order, from the directory it is told", () => { + const seen: string[] = []; + const load = (dir: string): Catalog => { + seen.push(dir); + return { ok: true, laws: [law("W-01"), law("W-02")] }; + }; + expect(yamlCatalog(load, "somewhere").map((entry) => entry.id)).toEqual(["W-01", "W-02"]); + expect(seen).toEqual(["somewhere"]); + yamlCatalog(load); + expect(seen[1]?.endsWith("/laws")).toBe(true); + }); + + it("a catalog with a file that is not valid fails with each problem, not an empty list", () => { + const bad: Catalog = { + ok: false, + problems: [ + { file: "W-01.yaml", pointer: "/id", message: "is required" }, + { file: "W-02.yaml", pointer: "", message: "is not a map" }, + ], + }; + expect(() => yamlCatalog(() => bad)).toThrow( + "the law catalog is invalid: W-01.yaml/id: is required; W-02.yaml: is not a map", + ); + }); +}); diff --git a/src/laws/layer/catalog.ts b/src/laws/layer/catalog.ts new file mode 100644 index 0000000..268bc25 --- /dev/null +++ b/src/laws/layer/catalog.ts @@ -0,0 +1,29 @@ +import { CATALOG_DIR, loadCatalog } from "../../laws/catalog/index.ts"; +import type { Law } from "../../laws/catalog/index.ts"; +import type { LawEntry } from "../../laws/readers/types.ts"; + +/** Where the layer reads its laws: a function, so a test gives it a catalog of its own. */ +export type LawCatalog = () => readonly LawEntry[]; + +/** A law as the layer takes it: what the catalog's file says, with a law that M3 gives a reader judged from now on. */ +export const entryOf = (law: Law): LawEntry => ({ + id: law.id, + severity: law.severity, + reader: law.reader.kinds[0] ?? "tbd", + classes: law.classes, + judged: law.judged !== "no", + rules: law.rules, +}); + +/** The kit's laws, from `laws/W-NN.yaml`: a catalog with a file that is not valid fails the layer, never an empty list. */ +export function yamlCatalog( + load: typeof loadCatalog = loadCatalog, + dir: string = CATALOG_DIR, +): readonly LawEntry[] { + const loaded = load(dir); + if (!loaded.ok) { + const lines = loaded.problems.map((found) => `${found.file}${found.pointer}: ${found.message}`); + throw new Error(`the law catalog is invalid: ${lines.join("; ")}`); + } + return loaded.laws.map(entryOf); +} diff --git a/src/laws/layer/entries.test.ts b/src/laws/layer/entries.test.ts new file mode 100644 index 0000000..1c1b96d --- /dev/null +++ b/src/laws/layer/entries.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { isRequired, lawsOfClass, toolsOf } from "./entries.ts"; +import type { LawEntry } from "../../laws/readers/types.ts"; + +const law = (id: string, extra: Partial = {}): LawEntry => ({ + id, + severity: "R", + reader: "tl", + classes: [], + judged: true, + rules: [], + ...extra, +}); + +describe("G4d which laws a class maps, and which it requires", () => { + it("a law with no class list is mapped by every class, and one with a list only by those classes", () => { + const laws = [law("W-01"), law("W-18", { classes: ["ts-library", "ui-components"] })]; + expect(lawsOfClass(laws, "ts-library").map((found) => found.id)).toEqual(["W-01", "W-18"]); + expect(lawsOfClass(laws, "node-service").map((found) => found.id)).toEqual(["W-01"]); + }); + + it("a law is required when it is judged and the class maps it", () => { + expect(isRequired(law("W-01"), "node-service")).toBe(true); + expect(isRequired(law("W-03", { judged: false }), "node-service")).toBe(false); + expect(isRequired(law("W-18", { classes: ["ts-library"] }), "node-service")).toBe(false); + expect(isRequired(law("W-18", { classes: ["ts-library"] }), "ts-library")).toBe(true); + }); + + it("names the tools a law maps rules of, once each", () => { + const rules = [ + { tool: "eslint", name: "a" }, + { tool: "tsc", name: "b" }, + { tool: "eslint", name: "c" }, + ]; + expect(toolsOf(law("W-13", { rules }))).toEqual(["eslint", "tsc"]); + expect(toolsOf(law("W-40"))).toEqual([]); + }); +}); diff --git a/src/laws/layer/entries.ts b/src/laws/layer/entries.ts new file mode 100644 index 0000000..ac260b7 --- /dev/null +++ b/src/laws/layer/entries.ts @@ -0,0 +1,17 @@ +import type { LawEntry } from "../../laws/readers/types.ts"; + +/** The laws a class maps: a law with no class list is mapped by every class (section 5.2). */ +export const lawsOfClass = (laws: readonly LawEntry[], className: string): LawEntry[] => + laws.filter((law) => law.classes.length === 0 || law.classes.includes(className)); + +/** + * A law is required when section 8.2 judges it and the class maps it (section 3.3). A mapped law that is not judged is + * counted and printed, never failing the run. + */ +export const isRequired = (law: LawEntry, className: string): boolean => + law.judged && lawsOfClass([law], className).length > 0; + +/** The tools a law maps rules of, in the order the law lists them, each once. */ +export const toolsOf = (law: LawEntry): string[] => [ + ...new Set(law.rules.map((rule) => rule.tool)), +]; diff --git a/src/laws/layer/index.test.ts b/src/laws/layer/index.test.ts new file mode 100644 index 0000000..73339da --- /dev/null +++ b/src/laws/layer/index.test.ts @@ -0,0 +1,230 @@ +import { symlinkSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cfg } from "../../gate/testkit.ts"; +import type { Exec, ExecResult, LayerContext } from "../../gate/types.ts"; +import { cleanTemp, put, tempDir } from "../../laws/probe/testkit.ts"; +import type { LawCatalog } from "./catalog.ts"; +import { lawsLayer } from "./index.ts"; +import type { LawEntry } from "../../laws/readers/types.ts"; + +afterAll(cleanTemp); + +const REPO = resolve(import.meta.dirname, "../../.."); +const ANY = "@typescript-eslint/no-explicit-any"; + +const law = (id: string, extra: Partial = {}): LawEntry => ({ + id, + severity: "R", + reader: "tl", + classes: [], + judged: true, + rules: [], + ...extra, +}); + +const W01 = law("W-01", { rules: [{ tool: "eslint", name: "no-explicit-any" }] }); +const W04 = law("W-04", { + reader: "an", + rules: [ + { tool: "tsc", name: "strict" }, + { tool: "tsc", name: "noUncheckedIndexedAccess" }, + ], +}); + +const TSCONFIG = + '{\n "compilerOptions": {\n "strict": true,\n "noUncheckedIndexedAccess": true\n },\n "include": ["src"]\n}\n'; + +interface Scenario { + readonly tracked: string[]; + /** ESLint's answer: each file's levels, and the ignored ones. */ + readonly levels: Record>; + readonly ignored?: Record; + readonly git?: ExecResult; +} + +const ok = (stdout: string): ExecResult => ({ code: 0, stdout, stderr: "" }); + +/** A fake process runner: `git ls-files` lists the tracked files, and `node` answers the ESLint question. */ +function runner(scenario: Scenario, calls: string[]): Exec { + return (command) => { + calls.push(command); + if (command === "git") { + return Promise.resolve(scenario.git ?? ok(`${scenario.tracked.join("\0")}\0`)); + } + const levels = scenario.levels; + const ignored = Object.fromEntries( + scenario.tracked.map((file) => [file, scenario.ignored?.[file] === true]), + ); + return Promise.resolve(ok(JSON.stringify({ ignored, levels }))); + }; +} + +function project(): string { + const cwd = tempDir(); + symlinkSync(join(REPO, "node_modules"), join(cwd, "node_modules")); + put(cwd, { + "tsconfig.json": TSCONFIG, + "src/a.ts": "export const a = 1;\n", + "out/eslint.json": "[]\n", + "out/tsc.json": "[]\n", + "eslint.config.js": `export default [\n { rules: { "${ANY}": "error" } },\n { files: ["src/**"], rules: { "${ANY}": "off" } },\n];\n`, + }); + return cwd; +} + +function context(cwd: string, exec: Exec, className = "ts-library"): LayerContext { + return { + cwd, + config: cfg({ class: className === "ts-library" ? "ts-library" : "node-service" }), + base: "origin/main", + enforce: "hard", + env: {}, + exec, + outDir: join(cwd, ".web-tops", "out"), + artifacts: new Map([ + ["eslint-json", join(cwd, "out/eslint.json")], + ["tsc-diagnostics", join(cwd, "out/tsc.json")], + ]), + }; +} + +const CATALOG = + (...laws: LawEntry[]): LawCatalog => + () => + laws; + +describe("G4a the laws layer is registered as the last gate, reading what hygiene wrote", () => { + it("is gate laws at tier 1, requires hygiene and consumes its ESLint JSON, and maps every launch class", () => { + const layer = lawsLayer(CATALOG()); + expect(layer).toMatchObject({ id: "laws", gate: "laws", tier: 1, requires: ["hygiene"] }); + expect(layer.consumes.filter((need) => need.optional !== true)).toEqual([ + { artifact: "eslint-json", from: "hygiene" }, + { artifact: "tsc-diagnostics", from: "hygiene" }, + ]); + expect( + layer.consumes.filter((need) => need.optional === true).map((need) => need.artifact), + ).toEqual([ + "attw-json", + "audit-json", + "dependency-cruiser-json", + "install-json", + "publint-json", + ]); + expect(layer.classes).toEqual([ + "ts-library", + "ui-components", + "astro-site", + "web-app", + "node-service", + ]); + }); +}); + +describe("G4 the laws layer reads each law's tools and gives every mapped law a finding", () => { + it("passes when the rules are at error for every file and the flags are set, with one finding for each law", async () => { + const cwd = project(); + const scenario: Scenario = { tracked: ["src/a.ts"], levels: { "src/a.ts": { [ANY]: 2 } } }; + const result = await lawsLayer(CATALOG(W01, W04)).run(context(cwd, runner(scenario, []))); + expect(result.verdict).toBe("pass"); + expect( + result.findings.map((found) => [found.rule, found.law, found.verdict, found.required]), + ).toEqual([ + ["law/W-01", "W-01", "pass", true], + ["law/W-04", "W-04", "pass", true], + ]); + }); + + it("an override that turns a required rule off is refuse at its config line, and ends the layer refuse", async () => { + const cwd = project(); + const scenario: Scenario = { tracked: ["src/a.ts"], levels: { "src/a.ts": { [ANY]: 0 } } }; + const result = await lawsLayer(CATALOG(W01)).run(context(cwd, runner(scenario, []))); + expect(result.verdict).toBe("refuse"); + expect( + result.findings.map((found) => [found.rule, found.verdict, found.file, found.line]), + ).toEqual([ + ["law/W-01", "refuse", undefined, undefined], + [ANY, "refuse", "eslint.config.js", 3], + ]); + }); + + it("a required law no reader reads is not-evaluated and ends the layer, where one not required only counts", async () => { + const cwd = project(); + const scenario: Scenario = { tracked: ["src/a.ts"], levels: { "src/a.ts": { [ANY]: 2 } } }; + const audit = law("W-38", { rules: [{ tool: "audit", name: "audit" }] }); + const required = await lawsLayer(CATALOG(W01, audit)).run(context(cwd, runner(scenario, []))); + expect(required.verdict).toBe("not-evaluated"); + expect(required.findings.find((found) => found.law === "W-38")?.message).toBe( + "W-38 not-evaluated: audit has no reader yet", + ); + const later = law("W-38", { judged: false, rules: [{ tool: "audit", name: "audit" }] }); + const counted = await lawsLayer(CATALOG(W01, later)).run(context(cwd, runner(scenario, []))); + expect(counted.verdict).toBe("pass"); + expect(counted.findings.find((found) => found.law === "W-38")).toMatchObject({ + verdict: "not-evaluated", + required: false, + }); + }); + + it("maps only the laws of the class, so a law the class does not map has no finding", async () => { + const cwd = project(); + const scenario: Scenario = { tracked: ["src/a.ts"], levels: { "src/a.ts": { [ANY]: 2 } } }; + const w18 = law("W-18", { + classes: ["ts-library"], + rules: [{ tool: "publint", name: "publint" }], + }); + const result = await lawsLayer(CATALOG(W01, w18)).run( + context(cwd, runner(scenario, []), "node-service"), + ); + expect(result.findings.map((found) => found.law)).toEqual(["W-01"]); + }); + + it("a tracked source file no ESLint config block reaches is a refusal of the layer's own, with its file", async () => { + const cwd = project(); + const scenario: Scenario = { + tracked: ["src/a.ts", "src/lost.mts"], + levels: { "src/a.ts": { [ANY]: 2 } }, + }; + const result = await lawsLayer(CATALOG(W01)).run(context(cwd, runner(scenario, []))); + expect(result.verdict).toBe("refuse"); + expect(result.findings.at(-1)).toMatchObject({ + tool: "eslint", + rule: "file-set", + verdict: "refuse", + file: "src/lost.mts", + }); + }); + + it("asks ESLint once for all the readers, and runs git once", async () => { + const cwd = project(); + const calls: string[] = []; + const scenario: Scenario = { tracked: ["src/a.ts"], levels: { "src/a.ts": { [ANY]: 2 } } }; + await lawsLayer(CATALOG(W01, W04)).run(context(cwd, runner(scenario, calls))); + expect(calls).toEqual(["git", "node"]); + }); + + it("a git that cannot list the tracked files fails the layer with that finding", async () => { + const cwd = project(); + const git: ExecResult = { code: 128, stdout: "", stderr: "not a git repository" }; + const scenario: Scenario = { tracked: [], levels: {}, git }; + const result = await lawsLayer(CATALOG(W01)).run(context(cwd, runner(scenario, []))); + expect(result.verdict).toBe("fail"); + expect(result.findings[0]?.rule).toBe("git-ls-files-failed"); + }); + + it("ESLint that cannot be asked fails the laws that read it, with the reason, and not the others", async () => { + const cwd = project(); + const broken: Exec = (command) => + Promise.resolve( + command === "git" + ? ok("src/a.ts\0") + : { code: 1, stdout: "", stderr: "config did not load" }, + ); + const result = await lawsLayer(CATALOG(W01, W04)).run(context(cwd, broken)); + expect(result.verdict).toBe("fail"); + expect(result.findings.map((found) => [found.law, found.verdict])).toEqual([ + ["W-01", "fail"], + ["W-04", "pass"], + ]); + }); +}); diff --git a/src/laws/layer/index.ts b/src/laws/layer/index.ts new file mode 100644 index 0000000..7173ba2 --- /dev/null +++ b/src/laws/layer/index.ts @@ -0,0 +1,150 @@ +import { ratchetHeld } from "../../baseline/state.ts"; +import type { EslintRead } from "../../census/eslint.ts"; +import type { Layer, LayerContext, LayerResult } from "../../gate/types.ts"; +import { askEslint } from "../../laws/readers/eslint-facts.ts"; +import { trackedSource, unlinted } from "../../laws/readers/files.ts"; +import type { RawFinding } from "../../report/findings.ts"; +import type { Verdict } from "../../report/types.ts"; +import { trackedFiles } from "../../layers/coverage/tracked.ts"; +import { lawsOfClass, toolsOf } from "./entries.ts"; +import type { LawCatalog } from "./catalog.ts"; +import { evidenceOf, judgeEntry, lawFinding, readingsOf, worse, type Judged } from "./judge.ts"; +import { optionalArtifacts, defaultReaders, artifactOf } from "./readers.ts"; +import type { LawEntry, LawReader, LawReading, ReadInput } from "../../laws/readers/types.ts"; + +/** The laws layer (WEB_TOPS.md sections 3.1 and 6.5): it reads what gate 1 wrote, and runs no lint a second time. */ + +function classes(): readonly string[] { + return ["ts-library", "ui-components", "astro-site", "web-app", "node-service"]; +} + +/** The ESLint question is asked once, the first time a reader needs it. */ +function once(ask: () => Promise): () => Promise { + let pending: Promise | undefined; + return () => { + pending ??= ask(); + return pending; + }; +} + +/** A law whose tool artifact gate 1 did not write: not evaluated, and the reason names the artifact. */ +const absent = (law: LawEntry, artifact: string, tool: string): LawReading => ({ + law: law.id, + verdict: "not-evaluated", + reason: `${artifact} is missing: gate 1 did not write it, so ${tool} cannot be read`, + findings: [], +}); + +/** The artifact of the law's rules of this tool that gate 1 did not write, if any. */ +function missingOf( + law: LawEntry, + tool: string, + have: ReadonlyMap, +): string | undefined { + return law.rules + .filter((rule) => rule.tool === tool) + .map(artifactOf) + .find((artifact) => artifact !== undefined && !have.has(artifact)); +} + +/** + * One reader's readings for the laws that map its tool; a tool no law maps is not asked, and a law whose artifact is + * missing is not read at all. + */ +async function readTool( + reader: LawReader, + tool: string, + laws: readonly LawEntry[], + input: Omit, +): Promise { + const mine = laws.filter((law) => toolsOf(law).includes(tool)); + const gone = new Map( + mine.map((law) => [law, missingOf(law, tool, input.ctx.artifacts)] as const), + ); + const ready = mine.filter((law) => gone.get(law) === undefined); + const absentReadings = mine.flatMap((law) => { + const artifact = gone.get(law); + return artifact === undefined ? [] : [absent(law, artifact, tool)]; + }); + return ready.length === 0 + ? absentReadings + : [...absentReadings, ...(await reader({ ...input, laws: ready }))]; +} + +/** The readings of every reader for the laws that map its tool, by law id. */ +async function readAll( + laws: readonly LawEntry[], + input: Omit, + readers: ReadonlyMap, +): Promise> { + const byLaw = new Map(); + for (const [tool, reader] of readers) { + for (const reading of await readTool(reader, tool, laws, input)) { + byLaw.set(reading.law, [...(byLaw.get(reading.law) ?? []), reading]); + } + } + return byLaw; +} + +function judgeAll( + laws: readonly LawEntry[], + className: string, + byLaw: ReadonlyMap, + read: ReadonlySet, +): Judged[] { + return laws.map((law) => { + const given = byLaw.get(law.id) ?? []; + const tools = new Set(toolsOf(law).filter((tool) => read.has(tool))); + return judgeEntry(law, className, readingsOf(law, given, tools)); + }); +} + +/** The layer's verdict: the worst of every required law and every finding of the layer's own; a law not required is only counted. */ +export function layerVerdict(judged: readonly Judged[], own: readonly RawFinding[]): Verdict { + const required = judged.filter((one) => one.required).map((one) => one.verdict); + return [...required, ...own.map((finding) => finding.verdict)].reduce(worse, "pass"); +} + +function resultOf(judged: readonly Judged[], own: readonly RawFinding[]): LayerResult { + const findings = [...judged.flatMap((one) => [lawFinding(one), ...evidenceOf(one)]), ...own]; + return { verdict: layerVerdict(judged, own), findings, artifacts: {} }; +} + +async function run( + catalog: LawCatalog, + readers: ReadonlyMap, + ctx: LayerContext, +): Promise { + const tracked = await trackedFiles(ctx); + if ("finding" in tracked) { + return { verdict: "fail", findings: [tracked.finding], artifacts: {} }; + } + const source = trackedSource(tracked.files); + const runner = { cwd: ctx.cwd, env: ctx.env, exec: ctx.exec, outDir: ctx.outDir }; + const eslint = once(() => askEslint(runner, source)); + const laws = lawsOfClass(catalog(), ctx.config.class); + const input = { ctx, tracked: tracked.files, held: ratchetHeld(ctx), eslint }; + const byLaw = await readAll(laws, input, readers); + const facts = await eslint(); + const own = facts.ok ? unlinted(source, facts.facts) : []; + return resultOf(judgeAll(laws, ctx.config.class, byLaw, new Set(readers.keys())), own); +} + +/** The inputs the laws layer reads of gate 1: ESLint's JSON and tsc's diagnostics, and each tool reader's own, optional. */ +const CONSUMES: Layer["consumes"] = [ + { artifact: "eslint-json", from: "hygiene" }, + { artifact: "tsc-diagnostics", from: "hygiene" }, + ...optionalArtifacts().map((artifact) => ({ artifact, from: "hygiene", optional: true })), +]; + +/** The laws layer over a catalog of laws and the readers of each tool's rules. */ +export const lawsLayer = (catalog: LawCatalog, readers = defaultReaders()): Layer => ({ + id: "laws", + gate: "laws", + tier: 1, + classes: classes(), + requires: ["hygiene"], + produces: [], + consumes: CONSUMES, + run: (ctx) => run(catalog, readers, ctx), +}); diff --git a/src/laws/layer/judge.test.ts b/src/laws/layer/judge.test.ts new file mode 100644 index 0000000..05b0951 --- /dev/null +++ b/src/laws/layer/judge.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { evidenceOf, judgeEntry, lawFinding, readingsOf, sentence, worse } from "./judge.ts"; +import type { LawEntry, LawReading } from "../../laws/readers/types.ts"; + +const law = (id: string, extra: Partial = {}): LawEntry => ({ + id, + severity: "R", + reader: "tl", + classes: [], + judged: true, + rules: [{ tool: "eslint", name: "r" }], + ...extra, +}); + +const reading = (verdict: LawReading["verdict"], extra: Partial = {}): LawReading => ({ + law: "W-01", + verdict, + findings: [], + ...extra, +}); + +const spot = { + tool: "eslint", + rule: "x/r", + law: "W-01", + verdict: "refuse", + message: "m", + file: "eslint.config.js", + line: 3, +} as const; + +describe("G4d one verdict for a law, from what its readers say", () => { + it("takes the worst of the verdicts, refuse first and pass last", () => { + expect(worse("pass", "refuse")).toBe("refuse"); + expect(worse("not-evaluated", "fail")).toBe("fail"); + expect(worse("ack-missing", "not-evaluated")).toBe("ack-missing"); + expect(worse("pass", "pass")).toBe("pass"); + }); + + it("a law passes only when every reader passes: a refusal beats a pass, and not-evaluated beats it too", () => { + expect( + judgeEntry(law("W-01"), "ts-library", [reading("pass"), reading("refuse")]).verdict, + ).toBe("refuse"); + expect( + judgeEntry(law("W-01"), "ts-library", [reading("pass"), reading("not-evaluated")]).verdict, + ).toBe("not-evaluated"); + expect(judgeEntry(law("W-01"), "ts-library", [reading("pass"), reading("pass")]).verdict).toBe( + "pass", + ); + }); + + it("a break of a law whose severity is A is ack-missing, not refuse", () => { + const soft = law("W-09", { severity: "A" }); + expect(judgeEntry(soft, "ts-library", [reading("refuse")]).verdict).toBe("ack-missing"); + expect(judgeEntry(soft, "ts-library", [reading("fail")]).verdict).toBe("fail"); + expect(judgeEntry(soft, "ts-library", [reading("not-evaluated")]).verdict).toBe( + "not-evaluated", + ); + expect(judgeEntry(soft, "ts-library", [reading("pass")]).verdict).toBe("pass"); + }); + + it("a law that is not required is counted: a break is not-evaluated, and a tool that failed is still fail", () => { + const later = law("W-03", { judged: false }); + expect(judgeEntry(later, "ts-library", [reading("refuse")])).toMatchObject({ + required: false, + verdict: "not-evaluated", + }); + expect(judgeEntry(later, "ts-library", [reading("ack-missing")]).verdict).toBe("not-evaluated"); + expect(judgeEntry(later, "ts-library", [reading("fail")]).verdict).toBe("fail"); + }); + + it("a law the class does not map is not required, whatever its tick", () => { + const other = law("W-18", { classes: ["ts-library"] }); + expect(judgeEntry(other, "node-service", [reading("refuse")])).toMatchObject({ + required: false, + verdict: "not-evaluated", + }); + }); +}); + +describe("G4d a tool with no reader is not-evaluated for the law, never a pass", () => { + it("adds a not-evaluated reading for each tool no reader read, naming the tool", () => { + const both = law("W-13", { + rules: [ + { tool: "eslint", name: "a" }, + { tool: "tsc", name: "b" }, + ], + }); + const given = [reading("pass")]; + const readings = readingsOf(both, given, new Set(["eslint"])); + expect(readings.map((found) => [found.verdict, found.reason])).toEqual([ + ["pass", undefined], + ["not-evaluated", "tsc has no reader yet"], + ]); + }); + + it("says why a law with no tool rules has no reading, by its reader kind", () => { + const none = (reader: LawEntry["reader"]): string | undefined => + readingsOf(law("W-41", { reader, rules: [] }), [], new Set())[0]?.reason; + expect(none("an")).toBe("W-41 has no reader yet"); + expect(none("tl")).toBe("W-41 has no reader yet"); + expect(none("at")).toBe("judged by attestation, and none is recorded"); + expect(none("tbd")).toBe("no reader is built for it yet"); + }); + + it("every tool read: the readings are what the readers gave, and nothing is added", () => { + const readings = readingsOf(law("W-01"), [reading("pass")], new Set(["eslint"])); + expect(readings).toEqual([reading("pass")]); + }); +}); + +describe("G4d each law is a finding of its own, so the counts can be read back", () => { + it("is tool webtops, rule law/W-NN, with the law's verdict and whether it is required, and no place", () => { + const judged = judgeEntry(law("W-01"), "ts-library", [reading("pass")]); + expect(lawFinding(judged)).toEqual({ + tool: "webtops", + rule: "law/W-01", + law: "W-01", + verdict: "pass", + message: "W-01 pass", + required: true, + }); + }); + + it("says in a sentence what the readers noted, why it was not evaluated, and how many places are listed", () => { + const judged = judgeEntry(law("W-01"), "ts-library", [ + reading("refuse", { note: "no-explicit-any 2", findings: [spot, spot] }), + reading("not-evaluated", { reason: "tsc has no reader yet" }), + ]); + expect(sentence(judged)).toBe( + "W-01 refuse: no-explicit-any 2; tsc has no reader yet; 2 place(s) listed", + ); + }); + + it("carries the places that break the law with the law's verdict and requirement", () => { + const judged = judgeEntry(law("W-01"), "ts-library", [reading("refuse", { findings: [spot] })]); + expect(evidenceOf(judged)).toEqual([{ ...spot, verdict: "refuse", required: true }]); + const later = judgeEntry(law("W-03", { judged: false }), "ts-library", [ + reading("refuse", { findings: [spot] }), + ]); + expect(evidenceOf(later)).toEqual([{ ...spot, verdict: "not-evaluated", required: false }]); + }); +}); diff --git a/src/laws/layer/judge.ts b/src/laws/layer/judge.ts new file mode 100644 index 0000000..5338365 --- /dev/null +++ b/src/laws/layer/judge.ts @@ -0,0 +1,114 @@ +import type { RawFinding } from "../../report/findings.ts"; +import type { Verdict } from "../../report/types.ts"; +import { isRequired, toolsOf } from "./entries.ts"; +import type { LawEntry, LawReading } from "../../laws/readers/types.ts"; + +/** Worst first: the one verdict a law has when its readers disagree. A law passes only when every reader passes. */ +function worstFirst(): readonly Verdict[] { + return ["refuse", "fail", "ack-missing", "not-evaluated", "pass"]; +} + +/** The worse of two verdicts. */ +export const worse = (left: Verdict, right: Verdict): Verdict => + worstFirst().find((verdict) => verdict === left || verdict === right) ?? left; + +/** Why a law with no tool rules to read has no reading: its reader is not built, or it is judged by review. */ +function unreadReason(law: LawEntry): string { + if (law.reader === "at") { + return "judged by attestation, and none is recorded"; + } + return law.reader === "tbd" ? "no reader is built for it yet" : `${law.id} has no reader yet`; +} + +const unread = (law: LawEntry, reason: string): LawReading => ({ + law: law.id, + verdict: "not-evaluated", + reason, + findings: [], +}); + +/** + * What every tool a law maps comes to: the readings its readers gave, and a `not-evaluated` for each tool with no + * reader. A law that maps no tool has one reading, saying why it has none. + */ +export function readingsOf( + law: LawEntry, + given: readonly LawReading[], + read: ReadonlySet, +): LawReading[] { + const tools = toolsOf(law); + if (tools.length === 0) { + return [unread(law, unreadReason(law))]; + } + const missing = tools.filter((tool) => !read.has(tool)); + return [...given, ...missing.map((tool) => unread(law, `${tool} has no reader yet`))]; +} + +/** A break of a law whose severity is `A` needs an acknowledgement; of one whose severity is `R`, it is a refusal. */ +const forSeverity = (law: LawEntry, verdict: Verdict): Verdict => + law.severity === "A" && verdict === "refuse" ? "ack-missing" : verdict; + +/** + * A law that is not required is counted and printed, and never ends the run: a break is `not-evaluated`, not a refusal + * (section 3.3). A tool that could not run is still a failure. + */ +function forRequirement(required: boolean, verdict: Verdict): Verdict { + return !required && (verdict === "refuse" || verdict === "ack-missing") + ? "not-evaluated" + : verdict; +} + +export interface Judged { + readonly law: LawEntry; + readonly required: boolean; + readonly verdict: Verdict; + readonly readings: readonly LawReading[]; +} + +/** One law's verdict over its readings, as its severity and the class's requirement make it. */ +export function judgeEntry( + law: LawEntry, + className: string, + readings: readonly LawReading[], +): Judged { + const required = isRequired(law, className); + const combined = readings.map((reading) => reading.verdict).reduce(worse, "pass"); + const verdict = forRequirement(required, forSeverity(law, combined)); + return { law, required, verdict, readings }; +} + +const notes = (readings: readonly LawReading[]): string[] => + readings.flatMap((reading) => (reading.note === undefined ? [] : [reading.note])); + +const reasons = (readings: readonly LawReading[]): string[] => + readings.flatMap((reading) => (reading.reason === undefined ? [] : [reading.reason])); + +/** The one sentence a law's finding says. */ +export function sentence(judged: Judged): string { + const { law, readings } = judged; + const places = readings.reduce((total, reading) => total + reading.findings.length, 0); + const detail = [...notes(readings), ...reasons(readings)].join("; "); + const where = places > 0 ? `${String(places)} place(s) listed` : ""; + const extra = [detail, where].filter((part) => part !== "").join("; "); + return extra === "" ? `${law.id} ${judged.verdict}` : `${law.id} ${judged.verdict}: ${extra}`; +} + +/** The finding that is the law's own verdict: one for every mapped law, so the counts can be read back from the findings. */ +export function lawFinding(judged: Judged): RawFinding { + return { + tool: "webtops", + rule: `law/${judged.law.id}`, + law: judged.law.id, + verdict: judged.verdict, + message: sentence(judged), + required: judged.required, + }; +} + +/** The places a law is broken, as findings that carry the law's verdict and whether it is required. */ +export function evidenceOf(judged: Judged): RawFinding[] { + const verdict = judged.verdict; + return judged.readings + .flatMap((reading) => reading.findings) + .map((finding) => ({ ...finding, verdict, required: judged.required })); +} diff --git a/src/laws/layer/optional.test.ts b/src/laws/layer/optional.test.ts new file mode 100644 index 0000000..8eb5d8d --- /dev/null +++ b/src/laws/layer/optional.test.ts @@ -0,0 +1,142 @@ +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { cfg } from "../../gate/testkit.ts"; +import type { Exec, LayerContext } from "../../gate/types.ts"; +import { cleanTemp, tempDir } from "../probe/testkit.ts"; +import type { LawEntry, LawReader } from "../readers/types.ts"; +import type { LawCatalog } from "./catalog.ts"; +import { lawsLayer } from "./index.ts"; + +afterAll(cleanTemp); + +/** R92: the tool artifacts are optional inputs; a law whose artifact gate 1 did not write is not-evaluated, never a pass. */ + +const law = (id: string, tool: string, name: string): LawEntry => ({ + id, + severity: "R", + reader: "tl", + classes: [], + judged: true, + rules: [{ tool, name }], +}); + +const AUDIT = law("W-38", "bun", "audit"); +const LOCK = law("W-39", "bun", "install --frozen-lockfile"); +const CRUISE = law("W-19", "dependency-cruiser", "no-circular"); + +const CATALOG = + (...laws: LawEntry[]): LawCatalog => + () => + laws; + +const exec: Exec = (command) => + Promise.resolve({ + code: 0, + stdout: command === "git" ? "" : JSON.stringify({ ignored: {}, levels: {} }), + stderr: "", + }); + +function context(have: readonly string[]): LayerContext { + const cwd = tempDir(); + return { + cwd, + config: cfg({ class: "ts-library" }), + base: "origin/main", + enforce: "hard", + env: {}, + exec, + outDir: join(cwd, ".web-tops", "out"), + artifacts: new Map(have.map((name) => [name, join(cwd, name)])), + }; +} + +/** A reader that passes every law it is given, and records which. */ +function passing(seen: string[]): LawReader { + return (input) => { + seen.push(...input.laws.map((one) => one.id)); + return Promise.resolve( + input.laws.map((one) => ({ law: one.id, verdict: "pass" as const, findings: [] })), + ); + }; +} + +describe("G4 a law whose tool artifact is missing is not-evaluated", () => { + it("is declared as optional inputs of hygiene, beside the two it always needs, and writes no artifact", () => { + const layer = lawsLayer(CATALOG()); + const consumes = layer.consumes; + expect(layer.produces).toEqual([]); + expect(consumes.map((one) => one.from)).toEqual(consumes.map(() => "hygiene")); + expect(consumes.filter((one) => one.optional !== true).map((one) => one.artifact)).toEqual([ + "eslint-json", + "tsc-diagnostics", + ]); + expect(consumes.filter((one) => one.optional === true).map((one) => one.artifact)).toEqual([ + "attw-json", + "audit-json", + "dependency-cruiser-json", + "install-json", + "publint-json", + ]); + }); + + it("asks a reader for the rules of its own tool only: another tool's missing artifact is not its concern", async () => { + const seen: string[] = []; + const mixed: LawEntry = { + ...law("W-40", "eslint", "no-explicit-any"), + rules: [ + { tool: "eslint", name: "no-explicit-any" }, + { tool: "bun", name: "audit" }, + ], + }; + const layer = lawsLayer(CATALOG(mixed), new Map([["eslint", passing(seen)]])); + await layer.run(context([])); + expect(seen).toEqual(["W-40"]); + }); + + it("does not call a reader that has no law left to read", async () => { + let calls = 0; + const counting: LawReader = () => { + calls += 1; + return Promise.resolve([]); + }; + await lawsLayer(CATALOG(AUDIT), new Map([["bun", counting]])).run(context([])); + expect(calls).toBe(0); + }); + + it("does not ask the reader, and says which artifact is missing and which tool cannot be read", async () => { + const seen: string[] = []; + const layer = lawsLayer(CATALOG(AUDIT), new Map([["bun", passing(seen)]])); + const result = await layer.run(context([])); + expect(seen).toEqual([]); + expect(result.verdict).toBe("not-evaluated"); + expect(result.findings[0]).toMatchObject({ rule: "law/W-38", verdict: "not-evaluated" }); + expect(JSON.stringify(result.findings)).toContain( + "audit-json is missing: gate 1 did not write it, so bun cannot be read", + ); + }); + + it("reads the laws whose artifact is there and not the ones whose artifact is not, in one tool", async () => { + const seen: string[] = []; + const layer = lawsLayer(CATALOG(AUDIT, LOCK), new Map([["bun", passing(seen)]])); + const result = await layer.run(context(["install-json"])); + expect(seen).toEqual(["W-39"]); + expect(result.findings.map((one) => [one.rule, one.verdict])).toEqual([ + ["law/W-38", "not-evaluated"], + ["law/W-39", "pass"], + ]); + }); + + it("reads every law when every artifact is there, and the verdict is pass", async () => { + const seen: string[] = []; + const layer = lawsLayer( + CATALOG(AUDIT, CRUISE), + new Map([ + ["bun", passing(seen)], + ["dependency-cruiser", passing(seen)], + ]), + ); + const result = await layer.run(context(["audit-json", "dependency-cruiser-json"])); + expect(seen).toEqual(["W-38", "W-19"]); + expect(result.verdict).toBe("pass"); + }); +}); diff --git a/src/laws/layer/readers.test.ts b/src/laws/layer/readers.test.ts new file mode 100644 index 0000000..969a932 --- /dev/null +++ b/src/laws/layer/readers.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { artifactOf, defaultReaders, optionalArtifacts } from "./readers.ts"; + +describe("G4 the artifact each tool's reader reads", () => { + it("is the JSON of attw, dependency-cruiser and publint, by the tool", () => { + expect(artifactOf({ tool: "attw", name: "profile" })).toBe("attw-json"); + expect(artifactOf({ tool: "dependency-cruiser", name: "no-circular" })).toBe( + "dependency-cruiser-json", + ); + expect(artifactOf({ tool: "publint", name: "pack" })).toBe("publint-json"); + }); + + it("follows the rule for bun, which runs two checks", () => { + expect(artifactOf({ tool: "bun", name: "audit" })).toBe("audit-json"); + expect(artifactOf({ tool: "bun", name: "install --frozen-lockfile" })).toBe("install-json"); + expect(artifactOf({ tool: "bun", name: "pack" })).toBeUndefined(); + }); + + it("is none for a rule read from gate 1's ESLint or compiler output, and for a rule of bun's by another tool's name", () => { + expect(artifactOf({ tool: "eslint", name: "no-explicit-any" })).toBeUndefined(); + expect(artifactOf({ tool: "tsc", name: "strict" })).toBeUndefined(); + expect(artifactOf({ tool: "eslint", name: "audit" })).toBeUndefined(); + expect(artifactOf({ tool: "knip", name: "install --frozen-lockfile" })).toBeUndefined(); + }); + + it("lists every optional artifact once, in name order", () => { + expect(optionalArtifacts()).toEqual([ + "attw-json", + "audit-json", + "dependency-cruiser-json", + "install-json", + "publint-json", + ]); + }); + + it("registers the ESLint and compiler readers and no other", () => { + expect([...defaultReaders().keys()]).toEqual(["eslint", "tsc"]); + }); +}); diff --git a/src/laws/layer/readers.ts b/src/laws/layer/readers.ts new file mode 100644 index 0000000..9f9b7e5 --- /dev/null +++ b/src/laws/layer/readers.ts @@ -0,0 +1,48 @@ +import { eslintReader } from "../../laws/readers/eslint.ts"; +import { tscReader } from "../../laws/readers/tsc.ts"; +import type { LawReader } from "../../laws/readers/types.ts"; + +/** + * The readers by the tool whose rules they read. A law that maps a tool with no reader here is `not-evaluated` for that + * tool, never a pass: the readers of dependency-cruiser, publint, attw, audit, the secrets scanner, the lock and knip + * (G9) register here, one line each. + */ +export function defaultReaders(): ReadonlyMap { + return new Map([ + ["eslint", eslintReader], + ["tsc", tscReader], + ]); +} + +/** + * The artifact of gate 1 each tool's reader reads. The laws layer consumes each as optional (R92): the hygiene step + * that writes it can be off for a class, or not built yet. A law that maps one of these rules while its artifact is + * missing is `not-evaluated`, so a required law whose input is missing ends the run instead of passing on nothing. + * `bun` runs two checks, so its artifact follows the rule: the audit, or the frozen install. + */ +function toolArtifacts(): ReadonlyMap { + return new Map([ + ["attw", "attw-json"], + ["dependency-cruiser", "dependency-cruiser-json"], + ["publint", "publint-json"], + ]); +} + +function bunArtifacts(): ReadonlyMap { + return new Map([ + ["audit", "audit-json"], + ["install --frozen-lockfile", "install-json"], + ]); +} + +/** The artifact a rule of a tool is read from, or undefined for a rule judged from gate 1's ESLint or compiler output. */ +export const artifactOf = (rule: { + readonly tool: string; + readonly name: string; +}): string | undefined => + rule.tool === "bun" ? bunArtifacts().get(rule.name) : toolArtifacts().get(rule.tool); + +/** Every artifact the laws layer may read, once each, in name order. */ +export function optionalArtifacts(): readonly string[] { + return [...new Set([...toolArtifacts().values(), ...bunArtifacts().values()])].sort(); +} diff --git a/src/layers/coverage/tracked.test.ts b/src/layers/coverage/tracked.test.ts new file mode 100644 index 0000000..51737ca --- /dev/null +++ b/src/layers/coverage/tracked.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { cfg } from "../../gate/testkit.ts"; +import type { ExecResult, LayerContext } from "../../gate/types.ts"; +import { trackedFiles } from "./tracked.ts"; + +function context(result: ExecResult, calls: unknown[][]): LayerContext { + return { + cwd: "/work", + config: cfg(), + base: "origin/main", + enforce: "hard", + env: { A: "1" }, + exec: (command, args, options) => { + calls.push([command, args, options]); + return Promise.resolve(result); + }, + outDir: "/work/out", + artifacts: new Map(), + }; +} + +describe("coverage: the files git tracks", () => { + it("lists them from git ls-files -z in the project, with no empty name for the trailing NUL", async () => { + const calls: unknown[][] = []; + const found = await trackedFiles( + context({ code: 0, stdout: "src/a.ts\0src/b.ts\0", stderr: "" }, calls), + ); + expect(found).toEqual({ files: ["src/a.ts", "src/b.ts"] }); + expect(calls).toEqual([["git", ["ls-files", "-z"], { cwd: "/work", env: { A: "1" } }]]); + }); + + it("names git in the finding when it is not installed, or did not finish", async () => { + const missing = await trackedFiles( + context({ code: 127, stdout: "", stderr: "", failure: "tool-missing" }, []), + ); + expect(missing).toMatchObject({ + finding: { rule: "tool-missing", message: "git is not installed or not on PATH" }, + }); + const slow = await trackedFiles( + context({ code: 1, stdout: "", stderr: "", failure: "tool-timeout" }, []), + ); + expect(slow).toMatchObject({ + finding: { rule: "tool-timeout", message: "git did not finish in time and was killed" }, + }); + }); + + it("names git ls-files and what it said when it exits non-zero", async () => { + const found = await trackedFiles( + context({ code: 128, stdout: "", stderr: "not a git repository" }, []), + ); + expect(found).toMatchObject({ + finding: { + tool: "webtops", + rule: "git-ls-files-failed", + message: "git ls-files exited 128: not a git repository", + }, + }); + }); +});