diff --git a/src/pyeval/_core.py b/src/pyeval/_core.py index c878693..06dfa89 100644 --- a/src/pyeval/_core.py +++ b/src/pyeval/_core.py @@ -69,48 +69,49 @@ def duration(self) -> float: """How long the task took to run, in seconds.""" return self.ctx.duration - def evaluate(self, evaluator: Evaluator) -> None: + def evaluate(self, *evaluators: Evaluator) -> None: results = _CURRENT_EVAL_RESULTS.get() if results is None: raise RuntimeError( "result.evaluate() called outside of an eval context. " "Did you call this eval function directly instead of running it via pytest?" ) - try: - raw = evaluator.evaluate_sync(self.ctx) - normalized = _EVALUATOR_OUTPUT_ADAPTER.validate_python(raw) - default_name = evaluator.get_default_evaluation_name() - mapping = ( - normalized - if isinstance(normalized, Mapping) - else {default_name: normalized} - ) - for name, result in mapping.items(): - if not isinstance(name, str): - name = str(name) - if not isinstance(result, EvaluationReason): - result = EvaluationReason( - value=result - if isinstance(result, bool | int | float | str) - else str(result) + for evaluator in evaluators: + try: + raw = evaluator.evaluate_sync(self.ctx) + normalized = _EVALUATOR_OUTPUT_ADAPTER.validate_python(raw) + default_name = evaluator.get_default_evaluation_name() + mapping = ( + normalized + if isinstance(normalized, Mapping) + else {default_name: normalized} + ) + for name, result in mapping.items(): + if not isinstance(name, str): + name = str(name) + if not isinstance(result, EvaluationReason): + result = EvaluationReason( + value=result + if isinstance(result, bool | int | float | str) + else str(result) + ) + results.append( + EvaluationResult( + name=name, + value=result.value, + reason=result.reason, + source=evaluator.as_spec(), + ) ) - results.append( - EvaluationResult( - name=name, - value=result.value, - reason=result.reason, + except Exception as e: + self.failures.append( + EvaluatorFailure( + name=evaluator.get_default_evaluation_name(), + error_message=f"{type(e).__name__}: {e}", + error_stacktrace=traceback.format_exc(), source=evaluator.as_spec(), ) ) - except Exception as e: - self.failures.append( - EvaluatorFailure( - name=evaluator.get_default_evaluation_name(), - error_message=f"{type(e).__name__}: {e}", - error_stacktrace=traceback.format_exc(), - source=evaluator.as_spec(), - ) - ) def _group_by_type( diff --git a/tests/evals/eval_extra_evaluators.py b/tests/evals/eval_extra_evaluators.py new file mode 100644 index 0000000..a8f2c3e --- /dev/null +++ b/tests/evals/eval_extra_evaluators.py @@ -0,0 +1,20 @@ +"""Regression test: case.evaluators can be unpacked into result.evaluate().""" + +from pyeval import Case, dataset, execute +from pyeval.evaluators import EqualsExpected, IsInstance, MaxDuration + + +@dataset( + Case(name="basic", inputs="hello", expected_output="hello"), + Case( + name="with_extra_evaluators", + inputs="hello", + expected_output="hello", + evaluators=(MaxDuration(seconds=1),), + ), +) +def eval_case_evaluators(case: Case): + result = execute(lambda x: x, case) + result.evaluate(EqualsExpected()) + result.evaluate(IsInstance(type_name="str")) + result.evaluate(*case.evaluators)