fix(diagnosis): require low relevancy before blaming short answers as off-topic

openagent_eval/diagnosis/blame.py

@@ -136,15 +136,23 @@ def _analyze_generation(
         failures: list[FailureInstance] = []
         gen_scores = scores.generation_scores
 
-        # Empty or very short answer with non-empty contexts
-        if scores.answer_length < ANSWER_TOO_SHORT and scores.context_count > 0:
+        # Empty or very short answer with non-empty contexts is only flagged
+        # when corroborated by low answer relevancy: legitimate short answers
+        # ("Yes.", "42", "Paris") are not failures on their own (issue #66).
+        relevancy = gen_scores.get("answer_relevancy", 1.0)
+        if (
+            scores.answer_length < ANSWER_TOO_SHORT
+            and scores.context_count > 0
+            and relevancy < LOW_RELEVANCY
+        ):
             failures.append(
                 FailureInstance(
                     mode=FailureMode.OFF_TOPIC_ANSWER,
                     blame=BlameTarget.GENERATION,
                     confidence=0.7,
-                    reason="Generated answer is empty or very short despite "
-                    "having retrieved contexts.",
+                    reason=f"Generated answer is empty or very short despite "
+                    f"having retrieved contexts, and answer relevancy is low "
+                    f"({relevancy:.2f}), corroborating a generation problem.",
                     question=question,
                     evidence=gen_scores,
                 )
@@ -166,7 +174,6 @@ def _analyze_generation(
             )
 
         # Low relevancy
-        relevancy = gen_scores.get("answer_relevancy", 1.0)
         if relevancy < LOW_RELEVANCY and scores.answer_length >= ANSWER_TOO_SHORT:
             failures.append(
                 FailureInstance(

tests/unit/test_diagnosis/test_blame.py

@@ -136,18 +136,61 @@ def test_low_relevancy(self) -> None:
         )
 
     def test_empty_answer_blames_generation(self) -> None:
-        """Empty answer with non-empty contexts should blame generation."""
+        """Short answer corroborated by low relevancy should blame generation."""
         scores = ComponentScores(
             question="What is AI?",
             retrieval_scores={"context_precision": 0.8},
-            generation_scores={},
+            generation_scores={"answer_relevancy": LOW_RELEVANCY - 0.1},
             context_count=3,
             context_lengths=[200, 200, 200],
             answer_length=5,  # Too short
         )
         result = self.blamer.analyze(scores)
         assert result.target == BlameTarget.GENERATION
 
+    def test_short_answer_with_good_scores_not_blamed(self) -> None:
+        """Legitimate short answers ("Yes.", "42", "Paris") must not be blamed.
+
+        Regression test for issue #66: a short answer corroborated as fine by
+        the other generation metrics is not an OFF_TOPIC_ANSWER.
+        """
+        for answer in ("Yes.", "42", "Paris"):
+            scores = ComponentScores(
+                question="What is the capital of France?",
+                retrieval_scores={"context_precision": 0.9, "context_recall": 0.85},
+                generation_scores={"faithfulness": 0.95, "answer_relevancy": 0.9},
+                context_count=3,
+                context_lengths=[200, 200, 200],
+                answer_length=len(answer),
+            )
+            result = self.blamer.analyze(scores)
+            assert result.target == BlameTarget.NONE, answer
+            assert not any(
+                f.mode == FailureMode.OFF_TOPIC_ANSWER for f in result.failure_modes
+            ), answer
+
+    def test_short_answer_with_low_relevancy_still_blamed(self) -> None:
+        """A short answer corroborated by low relevancy is still flagged.
+
+        Guards against the fix degenerating into disabling the heuristic.
+        """
+        scores = ComponentScores(
+            question="What is the capital of France?",
+            retrieval_scores={"context_precision": 0.9, "context_recall": 0.85},
+            generation_scores={
+                "faithfulness": 0.95,
+                "answer_relevancy": LOW_RELEVANCY - 0.1,
+            },
+            context_count=3,
+            context_lengths=[200, 200, 200],
+            answer_length=4,  # Too short, e.g. "idk."
+        )
+        result = self.blamer.analyze(scores)
+        assert result.target == BlameTarget.GENERATION
+        assert any(
+            f.mode == FailureMode.OFF_TOPIC_ANSWER for f in result.failure_modes
+        )
+
     # ------------------------------------------------------------------
     # Chunking failures
     # ------------------------------------------------------------------