Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ example (`promptproof run examples/multiturn.yaml --mock`).
| `max_chars` | `max_chars: 500` | the response is at most N characters |
| `min_chars` | `min_chars: 50` | the response is at least N characters |
| `equals` | `equals: "yes"` | the response matches exactly (whitespace-trimmed) |
| `not_equals` | `not_equals: "unknown"` | the response differs (whitespace-trimmed) |
| `starts_with` | `starts_with: "Summary:"` | the response begins with the literal prefix |
| `ends_with` | `ends_with: "done."` | the response ends with the literal suffix |
| `llm_judge` | `llm_judge: "the response is polite"` | a judge model decides the response meets the criterion |
Expand Down
14 changes: 12 additions & 2 deletions src/promptproof/assertions.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
- json_schema: {type: object, required: [sentiment]}
- llm_judge: "the response is polite and answers the question"

``equals`` compares after stripping leading/trailing whitespace (models love
trailing newlines); ``starts_with`` / ``ends_with`` match literally.
``equals`` and ``not_equals`` compare after stripping leading/trailing
whitespace (models love trailing newlines); ``starts_with`` / ``ends_with``
match literally.

``not_regex`` is the mirror of ``regex``: it passes when the pattern does
*not* match anywhere in the response — handy for banning PII shapes,
Expand Down Expand Up @@ -207,6 +208,14 @@ def _check_equals(response: str, expected: Any) -> AssertionResult:
)


def _check_not_equals(response: str, expected: Any) -> AssertionResult:
wanted = str(expected).strip()
ok = response.strip() != wanted
return AssertionResult(
"not_equals", ok, "" if ok else f"response equals forbidden {wanted!r}"
)


def _check_starts_with(response: str, expected: Any) -> AssertionResult:
prefix = str(expected)
ok = response.startswith(prefix)
Expand Down Expand Up @@ -274,6 +283,7 @@ def _check_llm_judge(response: str, expected: Any, judge: LLMJudge) -> Assertion
"max_chars": _check_max_chars,
"min_chars": _check_min_chars,
"equals": _check_equals,
"not_equals": _check_not_equals,
"starts_with": _check_starts_with,
"ends_with": _check_ends_with,
"json_schema": _check_json_schema,
Expand Down
33 changes: 33 additions & 0 deletions tests/test_assertions.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,39 @@ def test_equals():
assert not only("yes please", {"equals": "yes"}).passed


def test_not_equals():
result = only("yes please", {"not_equals": "yes"})
assert result.kind == "not_equals"
assert result.passed
assert result.detail == ""
result = only("yes", {"not_equals": "yes"})
assert not result.passed
assert "yes" in result.detail


def test_not_equals_trims_both_sides():
assert not only(" \tyes\n", {"not_equals": " yes "}).passed
assert not only("\n", {"not_equals": ""}).passed
assert only("Yes", {"not_equals": "yes"}).passed


def test_not_equals_explicit_form():
assert only("no", {"type": "not_equals", "value": "yes"}).passed
assert not only("42\n", {"type": "not_equals", "value": 42}).passed


@pytest.mark.parametrize(
"assertion, message",
[
({"type": "not_equals"}, "missing its 'value'"),
({"not_equals": "yes", "contains": "no"}, "exactly one key"),
],
)
def test_not_equals_malformed_config(assertion, message):
with pytest.raises(AssertionConfigError, match=message):
check_assertions("no", [assertion])


def test_starts_with():
assert only("Summary: great keyboard", {"starts_with": "Summary:"}).passed
assert not only("In summary: great", {"starts_with": "Summary:"}).passed
Expand Down
28 changes: 28 additions & 0 deletions tests/test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,34 @@ def test_json_suite_files_are_supported(tmp_path):
assert report.ok


def test_not_equals_from_yaml_suite(tmp_path):
suite_file = tmp_path / "not-equals.yaml"
suite_file.write_text(
"""
name: not-equals
prompt: "Answer the question"
cases:
- name: different answer
mock_response: "yes please"
assertions:
- not_equals: "yes"
- name: same answer
mock_response: " yes "
assertions:
- type: not_equals
value: "yes"
""",
encoding="utf-8",
)
report = run_suite(load_suite(suite_file), mock=True)
assert report.passed_count == 1
assert report.failed_count == 1
assert not report.ok
assert [case.assertions[0].kind for case in report.cases] == [
"not_equals", "not_equals"
]


def test_mock_rule_regex_matching(tmp_path):
suite_file = tmp_path / "suite.yaml"
suite_file.write_text(
Expand Down