Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/reporters/terminal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ export function renderRunTerminal(run: RunResult): string {
lines.push(bold(`\nevalgate - ${run.suite}`));
lines.push(dim(`${run.timestamp}`));
lines.push("");
if (run.total === 0) {
lines.push(red("No cases matched the filter."));
}

for (const cse of run.cases) {
const mark = cse.passed ? green("PASS") : red("FAIL");
Expand Down
3 changes: 2 additions & 1 deletion src/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ export async function runSuite(suite: EvalSuite, options: RunOptions = {}): Prom
const costUsd = caseResults.reduce((s, r) => s + r.costUsd, 0);

const thresholdMet = suite.threshold === undefined || meanScore >= suite.threshold;
const passed = thresholdMet && passedCount === total;
// A filtered run with no cases must fail instead of silently disabling the gate.
const passed = total > 0 && thresholdMet && passedCount === total;

return {
version: RESULT_VERSION,
Expand Down
5 changes: 5 additions & 0 deletions tests/reporters.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ describe("reporters", () => {
expect(md).toContain("| Case | Base | Head | Delta | Change |");
});

it("terminal reports explain when filtering selected no cases", async () => {
const run = await runSuite(suite, { filterTags: ["does-not-exist"] });
expect(renderRunTerminal(run)).toContain("No cases matched the filter");
});

it("terminal reports render without throwing", async () => {
const run = await runSuite(suite);
expect(renderRunTerminal(run)).toContain("evalgate");
Expand Down
7 changes: 7 additions & 0 deletions tests/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ describe("runSuite", () => {
expect(res.cases[0]!.id).toBe("ok");
});

it("fails when tag filtering selects no cases", async () => {
const suiteWithoutThreshold: EvalSuite = { ...suite, threshold: undefined };
const res = await runSuite(suiteWithoutThreshold, { filterTags: ["does-not-exist"] });
expect(res.total).toBe(0);
expect(res.passed).toBe(false);
});

it("fires the onCase callback per case", async () => {
const seen: string[] = [];
await runSuite(suite, { onCase: (c) => seen.push(c.id) });
Expand Down
Loading