fix: reject out-of-order voice timing

README.md

@@ -95,7 +95,7 @@ scripted test calls: in production this failure is silent, and no tool can fix t
 
 ## Honest scope
 
-- **The eval logic is the project, and it is fully tested** (18 tests, no keys, no network).
+- **The eval logic is the project, and it is fully tested** (20 tests, no keys, no network).
 - **The STT adapter is not exercised by the tests.** `GroqSTT` (whisper-large-v3, free tier) needs
   an API key and a network, and what is worth testing here is the evaluation, not whether Groq's
   SDK works. If your platform already gives you a timed transcript, you never need it.
@@ -117,7 +117,7 @@ From a clone, for development:
 
 ```bash
 pip install -e ".[dev]"
-pytest -q                                  # 18 tests
+pytest -q                                  # 20 tests
 ```
 
 Only dependency is `rich`. `[stt]` adds `groq` if you are starting from audio.

tests/test_voiceeval.py

@@ -20,7 +20,9 @@ def _codes(inter: Interaction) -> set[str]:
 
 
 def _t(speaker, text, start, end, truth=None, actions=None) -> Turn:
-    return Turn(speaker=speaker, text=text, start_s=start, end_s=end, truth=truth, actions=actions or [])
+    return Turn(
+        speaker=speaker, text=text, start_s=start, end_s=end, truth=truth, actions=actions or []
+    )
 
 
 # --------------------------------------------------------------------------- the headline case
@@ -112,7 +114,9 @@ def test_slow_response_is_caught():
         ],
     )
     findings = [f for f in analyse(inter) if f.check == "slow_response"]
-    assert findings and findings[0].severity == "high", "4s gap should be high, it is 2x+ the budget"
+    assert (
+        findings and findings[0].severity == "high"
+    ), "4s gap should be high, it is 2x+ the budget"
 
 
 def test_prompt_response_is_not_flagged():
@@ -122,6 +126,24 @@ def test_prompt_response_is_not_flagged():
     assert "slow_response" not in _codes(inter)
 
 
+def test_out_of_order_response_timing_is_caught():
+    """Corrupt clocks must fail loudly instead of looking like a fast response."""
+    inter = Interaction(
+        id="t",
+        turns=[
+            _t("user", "hello", 0, 2, truth="hello"),
+            _t("agent", "Hi.", 1.5, 2.0),
+        ],
+    )
+
+    findings = [f for f in analyse(inter) if f.check == "invalid_timing"]
+
+    assert len(findings) == 1
+    assert findings[0].severity == "high"
+    assert findings[0].turn_index == 1
+    assert "0.5s before" in findings[0].message
+
+
 def test_talking_over_user_is_caught():
     inter = Interaction(
         id="t",
@@ -174,8 +196,6 @@ def test_misheard_without_ground_truth_is_undetectable():
     assert "misheard_number" not in _codes(inter)
 
 
-
-
 def test_non_numeric_stt_mismatch_is_medium_misheard_not_number():
     """STT can mangle wording without touching amounts.
 
@@ -213,6 +233,7 @@ def test_non_numeric_stt_mismatch_is_medium_misheard_not_number():
     assert misheard[0].severity == "medium"
     assert misheard[0].turn_index == 0
 
+
 # --------------------------------------------------------------------------- regression diff
 
 

voiceeval/checks.py

@@ -21,7 +21,7 @@
 import re
 from dataclasses import dataclass
 
-from .turns import Interaction, Turn
+from .turns import Interaction
 
 
 @dataclass
@@ -44,9 +44,12 @@ class Finding:
     (r"\bfourteen\b", r"\bforty\b"),
 ]
 
-_NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|"
-                      r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|"
-                      r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I)
+_NUMERIC = re.compile(
+    r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|"
+    r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|"
+    r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b",
+    re.I,
+)
 
 _CONFIRM = re.compile(
     r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|"
@@ -82,7 +85,9 @@ def check_misheard(inter: Interaction) -> list[Finding]:
             )
         else:
             out.append(
-                Finding("misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i)
+                Finding(
+                    "misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i
+                )
             )
     return out
 
@@ -97,9 +102,7 @@ def check_acted_without_confirming(inter: Interaction) -> list[Finding]:
         consequential = [a for a in t.actions if a.consequential]
         if not consequential:
             continue
-        confirmed = any(
-            _CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent"
-        )
+        confirmed = any(_CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent")
         if not confirmed:
             names = ", ".join(a.name for a in consequential)
             out.append(
@@ -149,6 +152,17 @@ def check_latency(inter: Interaction, budget_s: float = 1.5) -> list[Finding]:
     out: list[Finding] = []
     for user_turn, agent_turn in inter.pairs():
         gap = agent_turn.start_s - user_turn.end_s
+        if gap < 0:
+            idx = inter.turns.index(agent_turn)
+            out.append(
+                Finding(
+                    "invalid_timing",
+                    "high",
+                    f"Agent started {abs(gap):.1f}s before the caller finished; timestamps overlap.",
+                    idx,
+                )
+            )
+            continue
         if gap > budget_s:
             idx = inter.turns.index(agent_turn)
             sev = "high" if gap > budget_s * 2 else "medium"
@@ -218,7 +232,9 @@ def analyse(inter: Interaction) -> list[Finding]:
     for check in CHECKS:
         out.extend(check(inter))
     order = {"high": 0, "medium": 1, "low": 2}
-    out.sort(key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1))
+    out.sort(
+        key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1)
+    )
     return out