Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,6 @@ unusable, so only `consequential` actions require it.

**Policy comes from the interaction, not this library.** What is allowed is a business decision.
Whether the agent respected it is the test.

Refund limits compare finite JSON numbers and numeric strings. Booleans, empty strings, malformed
amounts, and non-finite values are treated as missing amounts rather than valid refunds.
35 changes: 35 additions & 0 deletions tests/test_voiceeval.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,41 @@ def test_policy_violation_is_caught():
assert "policy_violation" in _codes(inter)


def test_policy_violation_is_caught_for_json_string_amount():
inter = Interaction(
id="t",
turns=[
_t(
"agent",
"Done.",
0,
1,
actions=[Action("refund", {"amount": " 200.50 "}, True)],
),
],
policy={"max_refund": 50},
)
assert "policy_violation" in _codes(inter)


def test_policy_violation_ignores_non_numeric_json_amounts():
for amount in (True, "", "not-a-number", "NaN", "Infinity", None):
inter = Interaction(
id="t",
turns=[
_t(
"agent",
"Done.",
0,
1,
actions=[Action("refund", {"amount": amount}, True)],
),
],
policy={"max_refund": 0},
)
assert "policy_violation" not in _codes(inter), repr(amount)


def test_slow_response_is_caught():
"""Invisible in a transcript. A four-second silence is a failed call."""
inter = Interaction(
Expand Down
23 changes: 21 additions & 2 deletions voiceeval/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import re
from dataclasses import dataclass
from decimal import Decimal, InvalidOperation

from .turns import Interaction, Turn

Expand Down Expand Up @@ -122,12 +123,18 @@ def check_policy_violation(inter: Interaction) -> list[Finding]:
"""
out: list[Finding] = []
max_refund = inter.policy.get("max_refund")
if max_refund is None:
normalized_max_refund = _finite_decimal(max_refund)
if normalized_max_refund is None:
return out
for i, t in enumerate(inter.turns):
for a in t.actions:
amount = a.args.get("amount")
if a.name == "refund" and isinstance(amount, (int, float)) and amount > max_refund:
normalized_amount = _finite_decimal(amount)
if (
a.name == "refund"
and normalized_amount is not None
and normalized_amount > normalized_max_refund
):
out.append(
Finding(
"policy_violation",
Expand All @@ -140,6 +147,18 @@ def check_policy_violation(inter: Interaction) -> list[Finding]:
return out


def _finite_decimal(value: object) -> Decimal | None:
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
return None
if isinstance(value, str) and not value.strip():
return None
try:
number = Decimal(str(value).strip())
except InvalidOperation:
return None
return number if number.is_finite() else None


def check_latency(inter: Interaction, budget_s: float = 1.5) -> list[Finding]:
"""Silence between the caller finishing and the agent starting.

Expand Down
Loading