From 39691e0e3ce1ea96bcde39735d9428389325f79d Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Tue, 14 Apr 2026 15:15:27 +0100 Subject: [PATCH 1/2] allow evaluate() to accept multiple cases to allow result.evaluate(*case.evaluators) pattern Co-Authored-By: Claude --- src/pyeval/_core.py | 65 ++++++++++++++-------------- tests/evals/eval_extra_evaluators.py | 20 +++++++++ 2 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 tests/evals/eval_extra_evaluators.py diff --git a/src/pyeval/_core.py b/src/pyeval/_core.py index c878693..06dfa89 100644 --- a/src/pyeval/_core.py +++ b/src/pyeval/_core.py @@ -69,48 +69,49 @@ def duration(self) -> float: """How long the task took to run, in seconds.""" return self.ctx.duration - def evaluate(self, evaluator: Evaluator) -> None: + def evaluate(self, *evaluators: Evaluator) -> None: results = _CURRENT_EVAL_RESULTS.get() if results is None: raise RuntimeError( "result.evaluate() called outside of an eval context. " "Did you call this eval function directly instead of running it via pytest?" ) - try: - raw = evaluator.evaluate_sync(self.ctx) - normalized = _EVALUATOR_OUTPUT_ADAPTER.validate_python(raw) - default_name = evaluator.get_default_evaluation_name() - mapping = ( - normalized - if isinstance(normalized, Mapping) - else {default_name: normalized} - ) - for name, result in mapping.items(): - if not isinstance(name, str): - name = str(name) - if not isinstance(result, EvaluationReason): - result = EvaluationReason( - value=result - if isinstance(result, bool | int | float | str) - else str(result) + for evaluator in evaluators: + try: + raw = evaluator.evaluate_sync(self.ctx) + normalized = _EVALUATOR_OUTPUT_ADAPTER.validate_python(raw) + default_name = evaluator.get_default_evaluation_name() + mapping = ( + normalized + if isinstance(normalized, Mapping) + else {default_name: normalized} + ) + for name, result in mapping.items(): + if not isinstance(name, str): + name = str(name) + if not isinstance(result, EvaluationReason): + result = EvaluationReason( + value=result + if isinstance(result, bool | int | float | str) + else str(result) + ) + results.append( + EvaluationResult( + name=name, + value=result.value, + reason=result.reason, + source=evaluator.as_spec(), + ) ) - results.append( - EvaluationResult( - name=name, - value=result.value, - reason=result.reason, + except Exception as e: + self.failures.append( + EvaluatorFailure( + name=evaluator.get_default_evaluation_name(), + error_message=f"{type(e).__name__}: {e}", + error_stacktrace=traceback.format_exc(), source=evaluator.as_spec(), ) ) - except Exception as e: - self.failures.append( - EvaluatorFailure( - name=evaluator.get_default_evaluation_name(), - error_message=f"{type(e).__name__}: {e}", - error_stacktrace=traceback.format_exc(), - source=evaluator.as_spec(), - ) - ) def _group_by_type( diff --git a/tests/evals/eval_extra_evaluators.py b/tests/evals/eval_extra_evaluators.py new file mode 100644 index 0000000..7bb3b72 --- /dev/null +++ b/tests/evals/eval_extra_evaluators.py @@ -0,0 +1,20 @@ +"""Regression test: case.evaluators can be unpacked into result.evaluate().""" + +from pyeval import Case, dataset, execute +from pyeval.evaluators import EqualsExpected, IsInstance, MaxDuration + + +@dataset( + Case(name="basic", inputs="hello", expected_output="hello"), + Case( + name="with_extra_evaluators", + inputs="hello", + expected_output="hello", + evaluators=[MaxDuration(seconds=1)], + ), +) +def eval_case_evaluators(case: Case): + result = execute(lambda x: x, case) + result.evaluate(EqualsExpected()) + result.evaluate(IsInstance(type_name="str")) + result.evaluate(*case.evaluators) From a4b05c8af6c6ee0c618b3721083eb07f38361a13 Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Tue, 14 Apr 2026 15:47:10 +0100 Subject: [PATCH 2/2] fix: use tuple for extra evaluators Co-Authored-By: Claude --- tests/evals/eval_extra_evaluators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/evals/eval_extra_evaluators.py b/tests/evals/eval_extra_evaluators.py index 7bb3b72..a8f2c3e 100644 --- a/tests/evals/eval_extra_evaluators.py +++ b/tests/evals/eval_extra_evaluators.py @@ -10,7 +10,7 @@ name="with_extra_evaluators", inputs="hello", expected_output="hello", - evaluators=[MaxDuration(seconds=1)], + evaluators=(MaxDuration(seconds=1),), ), ) def eval_case_evaluators(case: Case):