diff --git a/README.md b/README.md index 6ad30b1..3062ff3 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ scripted test calls: in production this failure is silent, and no tool can fix t ## Honest scope -- **The eval logic is the project, and it is fully tested** (18 tests, no keys, no network). +- **The eval logic is the project, and it is fully tested** (20 tests, no keys, no network). - **The STT adapter is not exercised by the tests.** `GroqSTT` (whisper-large-v3, free tier) needs an API key and a network, and what is worth testing here is the evaluation, not whether Groq's SDK works. If your platform already gives you a timed transcript, you never need it. @@ -117,7 +117,7 @@ From a clone, for development: ```bash pip install -e ".[dev]" -pytest -q # 18 tests +pytest -q # 20 tests ``` Only dependency is `rich`. `[stt]` adds `groq` if you are starting from audio. diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index b9aa363..51a79eb 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -20,7 +20,9 @@ def _codes(inter: Interaction) -> set[str]: def _t(speaker, text, start, end, truth=None, actions=None) -> Turn: - return Turn(speaker=speaker, text=text, start_s=start, end_s=end, truth=truth, actions=actions or []) + return Turn( + speaker=speaker, text=text, start_s=start, end_s=end, truth=truth, actions=actions or [] + ) # --------------------------------------------------------------------------- the headline case @@ -112,7 +114,9 @@ def test_slow_response_is_caught(): ], ) findings = [f for f in analyse(inter) if f.check == "slow_response"] - assert findings and findings[0].severity == "high", "4s gap should be high, it is 2x+ the budget" + assert ( + findings and findings[0].severity == "high" + ), "4s gap should be high, it is 2x+ the budget" def test_prompt_response_is_not_flagged(): @@ -122,6 +126,24 @@ def test_prompt_response_is_not_flagged(): assert "slow_response" not in _codes(inter) +def test_out_of_order_response_timing_is_caught(): + """Corrupt clocks must fail loudly instead of looking like a fast response.""" + inter = Interaction( + id="t", + turns=[ + _t("user", "hello", 0, 2, truth="hello"), + _t("agent", "Hi.", 1.5, 2.0), + ], + ) + + findings = [f for f in analyse(inter) if f.check == "invalid_timing"] + + assert len(findings) == 1 + assert findings[0].severity == "high" + assert findings[0].turn_index == 1 + assert "0.5s before" in findings[0].message + + def test_talking_over_user_is_caught(): inter = Interaction( id="t", @@ -174,8 +196,6 @@ def test_misheard_without_ground_truth_is_undetectable(): assert "misheard_number" not in _codes(inter) - - def test_non_numeric_stt_mismatch_is_medium_misheard_not_number(): """STT can mangle wording without touching amounts. @@ -213,6 +233,7 @@ def test_non_numeric_stt_mismatch_is_medium_misheard_not_number(): assert misheard[0].severity == "medium" assert misheard[0].turn_index == 0 + # --------------------------------------------------------------------------- regression diff diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..038399a 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -21,7 +21,7 @@ import re from dataclasses import dataclass -from .turns import Interaction, Turn +from .turns import Interaction @dataclass @@ -44,9 +44,12 @@ class Finding: (r"\bfourteen\b", r"\bforty\b"), ] -_NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" - r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" - r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I) +_NUMERIC = re.compile( + r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" + r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" + r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", + re.I, +) _CONFIRM = re.compile( r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|" @@ -82,7 +85,9 @@ def check_misheard(inter: Interaction) -> list[Finding]: ) else: out.append( - Finding("misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i) + Finding( + "misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i + ) ) return out @@ -97,9 +102,7 @@ def check_acted_without_confirming(inter: Interaction) -> list[Finding]: consequential = [a for a in t.actions if a.consequential] if not consequential: continue - confirmed = any( - _CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent" - ) + confirmed = any(_CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent") if not confirmed: names = ", ".join(a.name for a in consequential) out.append( @@ -149,6 +152,17 @@ def check_latency(inter: Interaction, budget_s: float = 1.5) -> list[Finding]: out: list[Finding] = [] for user_turn, agent_turn in inter.pairs(): gap = agent_turn.start_s - user_turn.end_s + if gap < 0: + idx = inter.turns.index(agent_turn) + out.append( + Finding( + "invalid_timing", + "high", + f"Agent started {abs(gap):.1f}s before the caller finished; timestamps overlap.", + idx, + ) + ) + continue if gap > budget_s: idx = inter.turns.index(agent_turn) sev = "high" if gap > budget_s * 2 else "medium" @@ -218,7 +232,9 @@ def analyse(inter: Interaction) -> list[Finding]: for check in CHECKS: out.extend(check(inter)) order = {"high": 0, "medium": 1, "low": 2} - out.sort(key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1)) + out.sort( + key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1) + ) return out