diff --git a/README.md b/README.md index 9421235..e020237 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,7 @@ example (`promptproof run examples/multiturn.yaml --mock`). | `max_chars` | `max_chars: 500` | the response is at most N characters | | `min_chars` | `min_chars: 50` | the response is at least N characters | | `equals` | `equals: "yes"` | the response matches exactly (whitespace-trimmed) | +| `not_equals` | `not_equals: "unknown"` | the response differs (whitespace-trimmed) | | `starts_with` | `starts_with: "Summary:"` | the response begins with the literal prefix | | `ends_with` | `ends_with: "done."` | the response ends with the literal suffix | | `llm_judge` | `llm_judge: "the response is polite"` | a judge model decides the response meets the criterion | diff --git a/src/promptproof/assertions.py b/src/promptproof/assertions.py index 4b7ceef..64473ea 100644 --- a/src/promptproof/assertions.py +++ b/src/promptproof/assertions.py @@ -12,8 +12,9 @@ - json_schema: {type: object, required: [sentiment]} - llm_judge: "the response is polite and answers the question" -``equals`` compares after stripping leading/trailing whitespace (models love -trailing newlines); ``starts_with`` / ``ends_with`` match literally. +``equals`` and ``not_equals`` compare after stripping leading/trailing +whitespace (models love trailing newlines); ``starts_with`` / ``ends_with`` +match literally. ``not_regex`` is the mirror of ``regex``: it passes when the pattern does *not* match anywhere in the response — handy for banning PII shapes, @@ -207,6 +208,14 @@ def _check_equals(response: str, expected: Any) -> AssertionResult: ) +def _check_not_equals(response: str, expected: Any) -> AssertionResult: + wanted = str(expected).strip() + ok = response.strip() != wanted + return AssertionResult( + "not_equals", ok, "" if ok else f"response equals forbidden {wanted!r}" + ) + + def _check_starts_with(response: str, expected: Any) -> AssertionResult: prefix = str(expected) ok = response.startswith(prefix) @@ -274,6 +283,7 @@ def _check_llm_judge(response: str, expected: Any, judge: LLMJudge) -> Assertion "max_chars": _check_max_chars, "min_chars": _check_min_chars, "equals": _check_equals, + "not_equals": _check_not_equals, "starts_with": _check_starts_with, "ends_with": _check_ends_with, "json_schema": _check_json_schema, diff --git a/tests/test_assertions.py b/tests/test_assertions.py index 252b949..f2a7a98 100644 --- a/tests/test_assertions.py +++ b/tests/test_assertions.py @@ -68,6 +68,39 @@ def test_equals(): assert not only("yes please", {"equals": "yes"}).passed +def test_not_equals(): + result = only("yes please", {"not_equals": "yes"}) + assert result.kind == "not_equals" + assert result.passed + assert result.detail == "" + result = only("yes", {"not_equals": "yes"}) + assert not result.passed + assert "yes" in result.detail + + +def test_not_equals_trims_both_sides(): + assert not only(" \tyes\n", {"not_equals": " yes "}).passed + assert not only("\n", {"not_equals": ""}).passed + assert only("Yes", {"not_equals": "yes"}).passed + + +def test_not_equals_explicit_form(): + assert only("no", {"type": "not_equals", "value": "yes"}).passed + assert not only("42\n", {"type": "not_equals", "value": 42}).passed + + +@pytest.mark.parametrize( + "assertion, message", + [ + ({"type": "not_equals"}, "missing its 'value'"), + ({"not_equals": "yes", "contains": "no"}, "exactly one key"), + ], +) +def test_not_equals_malformed_config(assertion, message): + with pytest.raises(AssertionConfigError, match=message): + check_assertions("no", [assertion]) + + def test_starts_with(): assert only("Summary: great keyboard", {"starts_with": "Summary:"}).passed assert not only("In summary: great", {"starts_with": "Summary:"}).passed diff --git a/tests/test_runner.py b/tests/test_runner.py index b97daa3..f277fcc 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -61,6 +61,34 @@ def test_json_suite_files_are_supported(tmp_path): assert report.ok +def test_not_equals_from_yaml_suite(tmp_path): + suite_file = tmp_path / "not-equals.yaml" + suite_file.write_text( + """ +name: not-equals +prompt: "Answer the question" +cases: + - name: different answer + mock_response: "yes please" + assertions: + - not_equals: "yes" + - name: same answer + mock_response: " yes " + assertions: + - type: not_equals + value: "yes" +""", + encoding="utf-8", + ) + report = run_suite(load_suite(suite_file), mock=True) + assert report.passed_count == 1 + assert report.failed_count == 1 + assert not report.ok + assert [case.assertions[0].kind for case in report.cases] == [ + "not_equals", "not_equals" + ] + + def test_mock_rule_regex_matching(tmp_path): suite_file = tmp_path / "suite.yaml" suite_file.write_text(