From 192a667712453b34f933542a1d99d11397e03270 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sun, 6 Sep 2026 16:12:45 +0900 Subject: [PATCH] fix: validate JSON refund amounts Signed-off-by: eunwoo song --- README.md | 3 +++ tests/test_voiceeval.py | 35 +++++++++++++++++++++++++++++++++++ voiceeval/checks.py | 23 +++++++++++++++++++++-- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ad30b1..7ac8aaf 100644 --- a/README.md +++ b/README.md @@ -135,3 +135,6 @@ unusable, so only `consequential` actions require it. **Policy comes from the interaction, not this library.** What is allowed is a business decision. Whether the agent respected it is the test. + +Refund limits compare finite JSON numbers and numeric strings. Booleans, empty strings, malformed +amounts, and non-finite values are treated as missing amounts rather than valid refunds. diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index b9aa363..e12cd92 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -102,6 +102,41 @@ def test_policy_violation_is_caught(): assert "policy_violation" in _codes(inter) +def test_policy_violation_is_caught_for_json_string_amount(): + inter = Interaction( + id="t", + turns=[ + _t( + "agent", + "Done.", + 0, + 1, + actions=[Action("refund", {"amount": " 200.50 "}, True)], + ), + ], + policy={"max_refund": 50}, + ) + assert "policy_violation" in _codes(inter) + + +def test_policy_violation_ignores_non_numeric_json_amounts(): + for amount in (True, "", "not-a-number", "NaN", "Infinity", None): + inter = Interaction( + id="t", + turns=[ + _t( + "agent", + "Done.", + 0, + 1, + actions=[Action("refund", {"amount": amount}, True)], + ), + ], + policy={"max_refund": 0}, + ) + assert "policy_violation" not in _codes(inter), repr(amount) + + def test_slow_response_is_caught(): """Invisible in a transcript. A four-second silence is a failed call.""" inter = Interaction( diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..3d01d8c 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -20,6 +20,7 @@ import re from dataclasses import dataclass +from decimal import Decimal, InvalidOperation from .turns import Interaction, Turn @@ -122,12 +123,18 @@ def check_policy_violation(inter: Interaction) -> list[Finding]: """ out: list[Finding] = [] max_refund = inter.policy.get("max_refund") - if max_refund is None: + normalized_max_refund = _finite_decimal(max_refund) + if normalized_max_refund is None: return out for i, t in enumerate(inter.turns): for a in t.actions: amount = a.args.get("amount") - if a.name == "refund" and isinstance(amount, (int, float)) and amount > max_refund: + normalized_amount = _finite_decimal(amount) + if ( + a.name == "refund" + and normalized_amount is not None + and normalized_amount > normalized_max_refund + ): out.append( Finding( "policy_violation", @@ -140,6 +147,18 @@ def check_policy_violation(inter: Interaction) -> list[Finding]: return out +def _finite_decimal(value: object) -> Decimal | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + if isinstance(value, str) and not value.strip(): + return None + try: + number = Decimal(str(value).strip()) + except InvalidOperation: + return None + return number if number.is_finite() else None + + def check_latency(inter: Interaction, budget_s: float = 1.5) -> list[Finding]: """Silence between the caller finishing and the agent starting.