fix: reject llm-judge scores outside [0, 1]

src/scorers/llm-judge.ts

@@ -1,5 +1,5 @@
 import type { Scorer, ScoreContext, ScorerSpec } from "../types.js";
-import { result } from "./util.js";
+import { clamp01, result } from "./util.js";
 
 /** Tokenize into a lowercase word set for the deterministic mock judge. */
 function wordSet(text: string): Set<string> {
@@ -96,7 +96,15 @@ export const llmJudgeScorer: Scorer = {
       reason = parsed.reason;
     }
 
-    const passed = score >= threshold;
-    return result(spec, { score, passed, reason });
+    if (score < 0 || score > 1) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `${reason} (judge score must be between 0 and 1)`,
+      });
+    }
+    const clamped = clamp01(score);
+    const passed = clamped >= threshold;
+    return result(spec, { score: clamped, passed, reason });
   },
 };

tests/scorers.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import type { EvalCase, ProviderResponse, ScoreContext, ScorerSpec } from "../src/types.js";
+import type { EvalCase, Provider, ProviderResponse, ScoreContext, ScorerSpec } from "../src/types.js";
 import { MockProvider } from "../src/providers/mock.js";
 import { exactMatchScorer } from "../src/scorers/exact-match.js";
 import { regexScorer } from "../src/scorers/regex.js";
@@ -162,6 +162,52 @@ describe("llm-judge (mock)", () => {
   });
 });
 
+describe("llm-judge (provider reply)", () => {
+  it("fails when the judge returns a score outside [0, 1]", async () => {
+    const judgeProvider: Provider = {
+      name: "stub",
+      async complete() {
+        return {
+          output: '{"score": 85, "reason": "looks fine"}',
+          latencyMs: 1,
+          model: "stub",
+        };
+      },
+    };
+    const c = ctx("wrong answer");
+    c.provider = judgeProvider;
+    const r = await run(
+      llmJudgeScorer,
+      { type: "llm-judge", criteria: "must be correct", threshold: 0.7 },
+      c,
+    );
+    expect(r.score).toBe(0);
+    expect(r.passed).toBe(false);
+  });
+
+  it("applies the pass threshold to the clamped in-range score", async () => {
+    const judgeProvider: Provider = {
+      name: "stub",
+      async complete() {
+        return {
+          output: '{"score": 0.85, "reason": "ok"}',
+          latencyMs: 1,
+          model: "stub",
+        };
+      },
+    };
+    const c = ctx("answer");
+    c.provider = judgeProvider;
+    const r = await run(
+      llmJudgeScorer,
+      { type: "llm-judge", criteria: "must be correct", threshold: 0.9 },
+      c,
+    );
+    expect(r.score).toBe(0.85);
+    expect(r.passed).toBe(false);
+  });
+});
+
 describe("latency and cost budgets", () => {
   it("latency passes within budget", async () => {
     const r = await run(latencyScorer, { type: "latency", budgetMs: 100 }, ctx("x", { latencyMs: 50 }));