diff --git a/src/reporters/terminal.ts b/src/reporters/terminal.ts index 27cc04b..316ecfe 100644 --- a/src/reporters/terminal.ts +++ b/src/reporters/terminal.ts @@ -18,6 +18,9 @@ export function renderRunTerminal(run: RunResult): string { lines.push(bold(`\nevalgate - ${run.suite}`)); lines.push(dim(`${run.timestamp}`)); lines.push(""); + if (run.total === 0) { + lines.push(red("No cases matched the filter.")); + } for (const cse of run.cases) { const mark = cse.passed ? green("PASS") : red("FAIL"); diff --git a/src/runner.ts b/src/runner.ts index 3ec8834..7f05e80 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -146,7 +146,8 @@ export async function runSuite(suite: EvalSuite, options: RunOptions = {}): Prom const costUsd = caseResults.reduce((s, r) => s + r.costUsd, 0); const thresholdMet = suite.threshold === undefined || meanScore >= suite.threshold; - const passed = thresholdMet && passedCount === total; + // A filtered run with no cases must fail instead of silently disabling the gate. + const passed = total > 0 && thresholdMet && passedCount === total; return { version: RESULT_VERSION, diff --git a/tests/reporters.test.ts b/tests/reporters.test.ts index 9089a85..2127268 100644 --- a/tests/reporters.test.ts +++ b/tests/reporters.test.ts @@ -34,6 +34,11 @@ describe("reporters", () => { expect(md).toContain("| Case | Base | Head | Delta | Change |"); }); + it("terminal reports explain when filtering selected no cases", async () => { + const run = await runSuite(suite, { filterTags: ["does-not-exist"] }); + expect(renderRunTerminal(run)).toContain("No cases matched the filter"); + }); + it("terminal reports render without throwing", async () => { const run = await runSuite(suite); expect(renderRunTerminal(run)).toContain("evalgate"); diff --git a/tests/runner.test.ts b/tests/runner.test.ts index 2bfaf46..79d0c64 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -63,6 +63,13 @@ describe("runSuite", () => { expect(res.cases[0]!.id).toBe("ok"); }); + it("fails when tag filtering selects no cases", async () => { + const suiteWithoutThreshold: EvalSuite = { ...suite, threshold: undefined }; + const res = await runSuite(suiteWithoutThreshold, { filterTags: ["does-not-exist"] }); + expect(res.total).toBe(0); + expect(res.passed).toBe(false); + }); + it("fires the onCase callback per case", async () => { const seen: string[] = []; await runSuite(suite, { onCase: (c) => seen.push(c.id) });