fix(graders): record refused verdicts from the LLM graders

bixbench/graders.py

@@ -56,11 +56,20 @@ class GradingFunction(BaseModel):
     """Base class for grading functions."""
 
     def _parse_grade_response(self, response: str) -> GradeType:
-        """Parse the grade from LLM response."""
+        """Parse the grade from LLM response.
+
+        The grading prompts ask for one of `correct`, `incorrect` or `refused`.
+        Anything else, including a missing or malformed <grade> tag, is graded
+        incorrect.
+        """
         match = re.search(r"<grade>\s*(.*?)\s*</grade>", response, re.DOTALL)
         grade = match[1].strip().lower() if match else None
 
-        return GradeType.CORRECT if grade == "correct" else GradeType.INCORRECT
+        if grade == GradeType.CORRECT:
+            return GradeType.CORRECT
+        if grade == GradeType.REFUSED:
+            return GradeType.REFUSED
+        return GradeType.INCORRECT
 
     async def _grade_str_verifier(
         self,

tests/test_utils.py

@@ -1,8 +1,9 @@
 import sys
+from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
-from bixbench.graders import MCQGrader
+from bixbench.graders import GradeType, GradingFunction, MCQGrader, OpenEndedGrader
 from bixbench.utils import (
     AnswerMode,
     compute_metrics,
@@ -109,6 +110,83 @@ async def test_grade_mcq_answer(
     assert grade_result.refusal == expected_refusal
 
 
+@pytest.mark.parametrize(
+    ("grader_response", "expected_grade_type"),
+    [
+        pytest.param("<grade>correct</grade>", GradeType.CORRECT, id="correct"),
+        pytest.param("<grade>incorrect</grade>", GradeType.INCORRECT, id="incorrect"),
+        pytest.param("<grade>refused</grade>", GradeType.REFUSED, id="refused"),
+        # Both grading prompts advertise `<grade> correct </grade>` as the example
+        # output, so padding and casing must not change the verdict.
+        pytest.param(
+            "<grade> correct </grade>", GradeType.CORRECT, id="correct_padded"
+        ),
+        pytest.param(
+            "Reasoning...\n<grade>\n  REFUSED\n</grade>\n",
+            GradeType.REFUSED,
+            id="refused_uppercase_multiline",
+        ),
+        # A verdict we cannot read is a formatting failure by the grading model,
+        # not an abstention by the answerer, so it stays incorrect.
+        pytest.param(
+            "The predicted answer is wrong.", GradeType.INCORRECT, id="missing_tag"
+        ),
+        pytest.param("<grade> refused", GradeType.INCORRECT, id="unclosed_tag"),
+        pytest.param(
+            "<grade> partially correct </grade>",
+            GradeType.INCORRECT,
+            id="unexpected_verdict",
+        ),
+    ],
+)
+def test_parse_grade_response(grader_response: str, expected_grade_type: GradeType):
+    grade_type = GradingFunction()._parse_grade_response(grader_response)
+
+    assert grade_type == expected_grade_type
+
+
+def test_refusal_scores_zero_like_an_incorrect_answer():
+    """Refusals score 0, so recording them cannot move accuracy or n_correct."""
+    assert GradeType.CORRECT.numeric_grade == 1
+    assert GradeType.INCORRECT.numeric_grade == 0
+    assert GradeType.REFUSED.numeric_grade == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("evaluation_mode", ["llm_verifier", "range_verifier"])
+@pytest.mark.parametrize(
+    ("grader_response", "expected_grade", "expected_correct", "expected_refusal"),
+    [
+        pytest.param("<grade> correct </grade>", 1, True, False, id="correct"),
+        pytest.param("<grade> incorrect </grade>", 0, False, False, id="incorrect"),
+        pytest.param("<grade> refused </grade>", 0, False, True, id="refused"),
+        pytest.param("no grade tag here", 0, False, False, id="missing_tag"),
+    ],
+)
+async def test_open_ended_grading_records_refusal(
+    evaluation_mode: str,
+    grader_response: str,
+    expected_grade: int,
+    expected_correct: bool,
+    expected_refusal: bool,
+):
+    """Both LLM grading paths must report a `refused` verdict as a refusal."""
+    llm_client = AsyncMock()
+    llm_client.call_single.return_value = MagicMock(text=grader_response)
+
+    grade_result = await OpenEndedGrader(
+        evaluation_mode=evaluation_mode, llm_client=llm_client
+    ).grade(
+        question="What is the capital of France?",
+        target="Paris",
+        predicted="I cannot answer this question.",
+    )
+
+    assert grade_result.grade == expected_grade
+    assert grade_result.correct is expected_correct
+    assert grade_result.refusal is expected_refusal
+
+
 @pytest.mark.parametrize(
     ("grades", "is_refused", "metrics"),
     [