feat: detect self-contradicting results

agentrace/checks.py

@@ -173,6 +173,33 @@ def check_absence_as_evidence(run: AgentRun) -> list[Finding]:
     return []
 
 
+def check_self_contradiction(run: AgentRun) -> list[Finding]:
+    """A zero/absence verdict contradicted by evidence in the same result."""
+    verdict = re.search(
+        r"\b(?:none found|no (?:open )?[a-z][a-z -]*|all (?:tests? )?passed)\b",
+        run.result,
+        re.IGNORECASE,
+    )
+    if not verdict:
+        return []
+
+    evidence = re.search(
+        r"(?m)^\s*(?:\d+[.)]\s+\S|\|.+\|\s*$)|\bTraceback \(most recent call last\):",
+        run.result,
+        re.IGNORECASE,
+    )
+    if not evidence:
+        return []
+
+    return [
+        Finding(
+            "self_contradiction",
+            "medium",
+            "Result's verdict conflicts with enumerated evidence in the same output.",
+            _context(run.result, verdict.start()),
+        )
+    ]
+
 def check_url_without_verification(run: AgentRun) -> list[Finding]:
     """Lots of links, no sign anything was opened.
 
@@ -301,6 +328,7 @@ def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]:
     check_destructive_command,
     check_unverified_claim,
     check_absence_as_evidence,
+    check_self_contradiction,
     check_url_without_verification,
     check_prompt_hygiene,
     check_unquantified,

tests/test_agentrace.py

@@ -219,6 +219,15 @@ def test_absence_as_evidence_is_caught():
     assert "absence_as_evidence" in _codes(r)
 
 
+def test_self_contradiction_is_caught_with_enumerated_evidence():
+    r = _run(result="No open issues:\n1. #12 auth bypass\n2. #14 token leak\n" + "x" * 200)
+    findings = [f for f in analyse(r) if f.check == "self_contradiction"]
+    assert findings and findings[0].severity == "medium"
+
+def test_self_contradiction_plain_absence_is_clean():
+    r = _run(result="None found after checking the issue tracker. " + "x" * 200)
+    assert "self_contradiction" not in _codes(r)
+
 def test_hedged_claim_is_caught_but_only_low():
     """Hedging is honest. The bug is the hedge getting flattened into fact downstream, so this is
     a note, not an alarm."""