From aadfdb85f5fd609b2beeaca90e4376c5c28512e7 Mon Sep 17 00:00:00 2001
From: "Sharad." <sharadvc@users.noreply.github.com>
Date: Sat, 12 Sep 2026 14:27:54 +0000
Subject: [PATCH] fix: reject out-of-range llm-judge scores before pass check

Fixes #35
---
 src/scorers/llm-judge.ts | 14 +++++++++---
 tests/scorers.test.ts    | 48 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/src/scorers/llm-judge.ts b/src/scorers/llm-judge.ts
index 1b23f0b..a011cd7 100644
--- a/src/scorers/llm-judge.ts
+++ b/src/scorers/llm-judge.ts
@@ -1,5 +1,5 @@
 import type { Scorer, ScoreContext, ScorerSpec } from "../types.js";
-import { result } from "./util.js";
+import { clamp01, result } from "./util.js";
 
 /** Tokenize into a lowercase word set for the deterministic mock judge. */
 function wordSet(text: string): Set<string> {
@@ -96,7 +96,15 @@ export const llmJudgeScorer: Scorer = {
       reason = parsed.reason;
     }
 
-    const passed = score >= threshold;
-    return result(spec, { score, passed, reason });
+    if (score < 0 || score > 1) {
+      return result(spec, {
+        score: 0,
+        passed: false,
+        reason: `${reason} (judge score must be between 0 and 1)`,
+      });
+    }
+    const clamped = clamp01(score);
+    const passed = clamped >= threshold;
+    return result(spec, { score: clamped, passed, reason });
   },
 };
diff --git a/tests/scorers.test.ts b/tests/scorers.test.ts
index 78bf373..45d62ee 100644
--- a/tests/scorers.test.ts
+++ b/tests/scorers.test.ts
@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import type { EvalCase, ProviderResponse, ScoreContext, ScorerSpec } from "../src/types.js";
+import type { EvalCase, Provider, ProviderResponse, ScoreContext, ScorerSpec } from "../src/types.js";
 import { MockProvider } from "../src/providers/mock.js";
 import { exactMatchScorer } from "../src/scorers/exact-match.js";
 import { regexScorer } from "../src/scorers/regex.js";
@@ -162,6 +162,52 @@ describe("llm-judge (mock)", () => {
   });
 });
 
+describe("llm-judge (provider reply)", () => {
+  it("fails when the judge returns a score outside [0, 1]", async () => {
+    const judgeProvider: Provider = {
+      name: "stub",
+      async complete() {
+        return {
+          output: '{"score": 85, "reason": "looks fine"}',
+          latencyMs: 1,
+          model: "stub",
+        };
+      },
+    };
+    const c = ctx("wrong answer");
+    c.provider = judgeProvider;
+    const r = await run(
+      llmJudgeScorer,
+      { type: "llm-judge", criteria: "must be correct", threshold: 0.7 },
+      c,
+    );
+    expect(r.score).toBe(0);
+    expect(r.passed).toBe(false);
+  });
+
+  it("applies the pass threshold to the clamped in-range score", async () => {
+    const judgeProvider: Provider = {
+      name: "stub",
+      async complete() {
+        return {
+          output: '{"score": 0.85, "reason": "ok"}',
+          latencyMs: 1,
+          model: "stub",
+        };
+      },
+    };
+    const c = ctx("answer");
+    c.provider = judgeProvider;
+    const r = await run(
+      llmJudgeScorer,
+      { type: "llm-judge", criteria: "must be correct", threshold: 0.9 },
+      c,
+    );
+    expect(r.score).toBe(0.85);
+    expect(r.passed).toBe(false);
+  });
+});
+
 describe("latency and cost budgets", () => {
   it("latency passes within budget", async () => {
     const r = await run(latencyScorer, { type: "latency", budgetMs: 100 }, ctx("x", { latencyMs: 50 }));