From ba3ad26fb497d50cbc3b69226f7d54d3768cb0a3 Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Wed, 15 Apr 2026 22:58:23 +0100 Subject: [PATCH 1/3] optionally send to logfire if configured Co-Authored-By: Claude --- pyproject.toml | 2 + src/pyeval/_logfire.py | 82 +++++++++++++++++++++++++++++++++++++++++ src/pyeval/plugin.py | 5 +++ tests/evals/conftest.py | 8 ++++ 4 files changed, 97 insertions(+) create mode 100644 src/pyeval/_logfire.py diff --git a/pyproject.toml b/pyproject.toml index d50ac40..cb567a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requires-python = ">=3.10" dependencies = [ "pytest>=8.0", "pydantic-evals>=1.67", + "logfire-api>=3.14.1", ] [project.optional-dependencies] @@ -55,6 +56,7 @@ dev = [ "poethepoet>=0.42.1", "ruff>=0.15.5", "ty>=0.0.21", + "logfire", ] [tool.poe.tasks] diff --git a/src/pyeval/_logfire.py b/src/pyeval/_logfire.py new file mode 100644 index 0000000..d4dd83c --- /dev/null +++ b/src/pyeval/_logfire.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import logfire_api +from pydantic import TypeAdapter +from pydantic_evals.evaluators import EvaluationResult +from pydantic_evals.reporting import EvaluationReport, ReportCase, ReportCaseAggregate + +_logfire = logfire_api.Logfire(otel_scope="pytest-pyeval") + +_evaluation_results_adapter: TypeAdapter[Mapping[str, EvaluationResult]] = TypeAdapter( + Mapping[str, EvaluationResult] +) + + +def send_report(report: EvaluationReport) -> None: + """Send an EvaluationReport to Logfire as a span hierarchy. + + Creates an experiment-level span containing a child span per case, + mirroring the span structure produced by pydantic-evals' ``Dataset.evaluate()``. + + Spans are no-ops when Logfire is not configured. + """ + if not report.cases and not report.failures: + return + + n_cases = len(report.cases) + len(report.failures) + + with _logfire.span( + "evaluate {name}", + name=report.name, + n_cases=n_cases, + ) as experiment_span: + experiment_span.set_attribute("gen_ai.operation.name", "experiment") + for case in report.cases: + _send_case(case) + + averages = ReportCaseAggregate.average(report.cases) + if averages.assertions is not None: + experiment_span.set_attribute("assertion_pass_rate", averages.assertions) + + full_metadata: dict[str, Any] = { + "n_cases": n_cases, + "averages": averages.model_dump(), + } + experiment_span.set_attribute("logfire.experiment.metadata", full_metadata) + + +def _send_case(case: ReportCase) -> None: + case_attrs: dict[str, Any] = {"inputs": case.inputs} + if case.metadata is not None: + case_attrs["metadata"] = case.metadata + if case.expected_output is not None: + case_attrs["expected_output"] = case.expected_output + + with _logfire.span( + "case: {case_name}", case_name=case.name, **case_attrs + ) as case_span: + if case.output is not None: + case_span.set_attribute("output", case.output) + case_span.set_attribute("task_duration", case.task_duration) + if case.metrics: + case_span.set_attribute("metrics", case.metrics) + if case.attributes: + case_span.set_attribute("attributes", case.attributes) + if case.assertions: + case_span.set_attribute( + "assertions", + _evaluation_results_adapter.dump_python(case.assertions), + ) + if case.scores: + case_span.set_attribute( + "scores", + _evaluation_results_adapter.dump_python(case.scores), + ) + if case.labels: + case_span.set_attribute( + "labels", + _evaluation_results_adapter.dump_python(case.labels), + ) diff --git a/src/pyeval/plugin.py b/src/pyeval/plugin.py index 34c10c1..9e61488 100644 --- a/src/pyeval/plugin.py +++ b/src/pyeval/plugin.py @@ -14,6 +14,9 @@ from pydantic_evals.evaluators import EvaluatorFailure from pydantic_evals.reporting import EvaluationReport, ReportCase, ReportCaseFailure + +from pyeval._logfire import send_report + from ._core import ( _CURRENT_EVAL_RESULTS, _CURRENT_EXECUTION_RESULT, @@ -141,6 +144,8 @@ def teardown(self): ) report.print() + send_report(report) + class EvalItem(pytest.Item): def __init__(self, name: str, parent: Node, func: Callable[..., Any], case: Case): diff --git a/tests/evals/conftest.py b/tests/evals/conftest.py index a787c6d..461846b 100644 --- a/tests/evals/conftest.py +++ b/tests/evals/conftest.py @@ -1,6 +1,14 @@ +import logfire import pytest +@pytest.fixture(scope="session", autouse=True) +def configure_logfire(): + logfire.configure( + send_to_logfire="if-token-present", + ) + + @pytest.fixture def prefix(): return "HELLO" From 21d1c8ca1cc8e99ea2b9cad6e026b97e89cb02a3 Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Wed, 15 Apr 2026 23:03:40 +0100 Subject: [PATCH 2/3] update README Co-Authored-By: Claude --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index f555be7..3af5a80 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,33 @@ pyeval # discover and run all evals in the project pyeval evals/ # run evals under a specific path pyeval evals/eval_foo.py # run a single eval file ``` + +## Logfire integration + +`pytest-pyeval` automatically sends evaluation results to [Logfire](https://logfire.pydantic.dev/) +as experiment traces when Logfire is configured. + +Configure Logfire before your evals run using a session-scoped autouse fixture +in your `conftest.py`: + +```python +# tests/evals/conftest.py +import logfire +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def configure_logfire(): + logfire.configure( + send_to_logfire="if-token-present", + ) +``` + +That's it! With `LOGFIRE_TOKEN` set in your environment, evaluation traces will +appear in the Logfire web UI under the **Evals** view. + +To install Logfire: + +```shell +uv add --dev logfire +``` From 12e308b06e909a6d3fd3761a6db556b3620888d8 Mon Sep 17 00:00:00 2001 From: Alex Ward Date: Wed, 15 Apr 2026 23:10:02 +0100 Subject: [PATCH 3/3] format --- src/pyeval/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pyeval/plugin.py b/src/pyeval/plugin.py index 9e61488..0e89087 100644 --- a/src/pyeval/plugin.py +++ b/src/pyeval/plugin.py @@ -14,7 +14,6 @@ from pydantic_evals.evaluators import EvaluatorFailure from pydantic_evals.reporting import EvaluationReport, ReportCase, ReportCaseFailure - from pyeval._logfire import send_report from ._core import (