diff --git a/src/pragmata/api/eval.py b/src/pragmata/api/eval.py index fa031e99..57efe4a0 100644 --- a/src/pragmata/api/eval.py +++ b/src/pragmata/api/eval.py @@ -3,12 +3,14 @@ from pathlib import Path from typing import Any +from pragmata.core.eval.export import export_eval_train_meta from pragmata.core.eval.imports import import_eval_train_frame from pragmata.core.eval.tlmtc_adapters import run_tlmtc_train from pragmata.core.eval.transforms import build_tlmtc_frame -from pragmata.core.paths.eval_paths import resolve_eval_train_paths +from pragmata.core.paths.eval_paths import resolve_eval_train_meta_path, resolve_eval_train_paths from pragmata.core.paths.paths import WorkspacePaths from pragmata.core.schemas.annotation_task import Task +from pragmata.core.schemas.eval_output import EvalTrainMeta from pragmata.core.settings.eval_settings import EvalTrainSettings from pragmata.core.settings.settings_base import UNSET, Unset, load_config_file @@ -82,8 +84,9 @@ def train_evaluator( "train_kwargs": train_kwargs, }, ) + workspace = WorkspacePaths.from_base_dir(settings.base_dir) train_paths = resolve_eval_train_paths( - workspace=WorkspacePaths.from_base_dir(settings.base_dir), + workspace=workspace, task=settings.task, labeled_data_path=settings.labeled_data_path, export_id=settings.export_id, @@ -100,7 +103,7 @@ def train_evaluator( ) assert settings.target_name is not None - return run_tlmtc_train( + result = run_tlmtc_train( labeled_data=tlmtc_frame, work_dir=train_paths.tool_root, target_name=settings.target_name, @@ -111,3 +114,16 @@ def train_evaluator( trust_remote_code=settings.trust_remote_code, train_kwargs=settings.train_kwargs, ) + export_eval_train_meta( + meta=EvalTrainMeta( + run_id=result.paths.run_id, + task=settings.task, + annotation_export_id=train_paths.annotation_export_id, + ), + path=resolve_eval_train_meta_path( + workspace=workspace, + run_id=result.paths.run_id, + ), + ) + + return result diff --git a/src/pragmata/core/eval/export.py b/src/pragmata/core/eval/export.py new file mode 100644 index 00000000..e1729947 --- /dev/null +++ b/src/pragmata/core/eval/export.py @@ -0,0 +1,22 @@ +"""Export evaluator artifacts to disk.""" + +from pathlib import Path + +from pragmata.core.atomic_io import atomic_write_json +from pragmata.core.schemas.eval_output import EvalTrainMeta + + +def export_eval_train_meta( + meta: EvalTrainMeta, + path: Path, +) -> None: + """Write Pragmata-owned evaluator training metadata to disk as JSON. + + Args: + meta: Validated evaluator training metadata to persist. + path: Destination path for the JSON sidecar. + """ + if not path.parent.is_dir(): + raise FileNotFoundError(f"Eval train metadata parent directory does not exist: {path.parent}") + + atomic_write_json(meta.model_dump(mode="json"), path) diff --git a/src/pragmata/core/paths/eval_paths.py b/src/pragmata/core/paths/eval_paths.py index 3a2b0df8..5f05c82a 100644 --- a/src/pragmata/core/paths/eval_paths.py +++ b/src/pragmata/core/paths/eval_paths.py @@ -160,6 +160,24 @@ def resolve_eval_train_paths( ) +def resolve_eval_train_meta_path( + *, + workspace: WorkspacePaths, + run_id: str, +) -> Path: + """Build the Pragmata-owned metadata path for a completed eval train run. + + Args: + workspace: Workspace path bundle. + run_id: tlmtc evaluator training run identifier. + + Returns: + Path to the Pragmata train metadata sidecar under the tlmtc train-run + directory. + """ + return workspace.tool_root("eval") / "train_outputs" / run_id / "pragmata_train.meta.json" + + @dataclass(frozen=True, slots=True) class EvalScorePaths: """Path bundle for an eval score run. diff --git a/src/pragmata/core/schemas/eval_output.py b/src/pragmata/core/schemas/eval_output.py index aa2b0714..48cb6e03 100644 --- a/src/pragmata/core/schemas/eval_output.py +++ b/src/pragmata/core/schemas/eval_output.py @@ -1,6 +1,6 @@ -"""Output schemas for eval score artifacts.""" +"""Output schemas for eval artifacts.""" -from datetime import datetime +from datetime import UTC, datetime from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, PositiveInt @@ -10,6 +10,17 @@ type Rate = Annotated[float, Field(ge=0.0, le=1.0)] +class EvalTrainMeta(BaseModel): + """Pragmata-owned metadata for a completed evaluator training run.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + run_id: str + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + task: Task + annotation_export_id: str | None = None + + class RetrievalScoreReport(BaseModel): """Schema for retrieval_scores.json.""" diff --git a/tests/integration/test_eval.py b/tests/integration/test_eval.py index 5627b627..8f07f189 100644 --- a/tests/integration/test_eval.py +++ b/tests/integration/test_eval.py @@ -1,5 +1,6 @@ """Integration tests for the public evaluator training surface.""" +import json from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -302,6 +303,23 @@ def _assert_prepared_splits( assert not any(str(value).startswith(f"{unexpected_marker}: ") for value in text_values) +def _assert_pragmata_train_meta( + *, + result: Any, + run_id: str, + annotation_export_id: str | None, +) -> None: + """Assert Pragmata persisted train-run metadata beside tlmtc artifacts.""" + meta_path = result.paths.run_dir / "pragmata_train.meta.json" + + assert meta_path.is_file() + meta = json.loads(meta_path.read_text(encoding="utf-8")) + assert meta["run_id"] == run_id + assert meta["task"] == "retrieval" + assert meta["annotation_export_id"] == annotation_export_id + assert isinstance(meta["created_at"], str) + + class TestTrainEvaluator: """Integration tests for evaluator training.""" @@ -327,6 +345,11 @@ def test_runs_tlmtc_from_direct_labeled_csv( expected_marker="direct", ) assert result.paths.train_run_meta_path.is_file() + _assert_pragmata_train_meta( + result=result, + run_id="direct-input", + annotation_export_id=None, + ) assert result.paths.model_dir.is_dir() assert any(result.paths.model_dir.iterdir()) @@ -362,6 +385,11 @@ def test_uses_explicit_annotation_export( expected_marker="selected", unexpected_marker="unselected", ) + _assert_pragmata_train_meta( + result=result, + run_id="explicit-export", + annotation_export_id="export-a", + ) def test_uses_latest_annotation_export_for_task( self, @@ -394,3 +422,8 @@ def test_uses_latest_annotation_export_for_task( expected_marker="newer", unexpected_marker="older", ) + _assert_pragmata_train_meta( + result=result, + run_id="latest-export", + annotation_export_id="newer-export", + ) diff --git a/tests/unit/api/test_eval.py b/tests/unit/api/test_eval.py index a9f189e7..f3c8004e 100644 --- a/tests/unit/api/test_eval.py +++ b/tests/unit/api/test_eval.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from pathlib import Path from textwrap import dedent +from types import SimpleNamespace from typing import Any import pandas as pd @@ -13,6 +14,12 @@ from pragmata.core.schemas.annotation_task import Task +def _train_result( + run_id: str = "train-run-1", +) -> SimpleNamespace: + return SimpleNamespace(paths=SimpleNamespace(run_id=run_id)) + + def test_train_evaluator_orchestrates_direct_labeled_input( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -22,7 +29,7 @@ def test_train_evaluator_orchestrates_direct_labeled_input( input_csv.write_text("placeholder\nvalue\n", encoding="utf-8") eval_frame = pd.DataFrame({"source": ["eval"]}) tlmtc_frame = pd.DataFrame({"text": ["query"], "text_pair": ["chunk"], "label_relevant": [1]}) - expected_result = object() + expected_result = _train_result() calls: dict[str, Any] = {} def import_eval_train_frame( @@ -51,6 +58,7 @@ def run_tlmtc_train( monkeypatch.setattr(eval_api, "import_eval_train_frame", import_eval_train_frame) monkeypatch.setattr(eval_api, "build_tlmtc_frame", build_tlmtc_frame) monkeypatch.setattr(eval_api, "run_tlmtc_train", run_tlmtc_train) + monkeypatch.setattr(eval_api, "export_eval_train_meta", lambda **kwargs: calls.setdefault("export", kwargs)) result = eval_api.train_evaluator( labeled_data_path=input_csv, @@ -75,6 +83,11 @@ def run_tlmtc_train( "trust_remote_code": True, "train_kwargs": {"run_id": "train-run-1", "verbosity": "quiet"}, } + assert calls["export"]["meta"].model_dump(exclude={"created_at"}) == { + "run_id": "train-run-1", + "task": Task.RETRIEVAL, + "annotation_export_id": None, + } assert (tmp_path / "eval").is_dir() @@ -104,15 +117,17 @@ def test_train_evaluator_combines_config_and_explicit_overrides( ) tlmtc_frame = pd.DataFrame({"text": ["answer"], "text_pair": ["context"], "label_support_present": [1]}) train_calls: list[dict[str, Any]] = [] + expected_result = _train_result("override-run") monkeypatch.setattr(eval_api, "import_eval_train_frame", lambda *, path, task: pd.DataFrame({"path": [path]})) monkeypatch.setattr(eval_api, "build_tlmtc_frame", lambda frame, *, task, mode: tlmtc_frame) + monkeypatch.setattr(eval_api, "export_eval_train_meta", lambda **kwargs: None) def run_tlmtc_train( **kwargs: Any, - ) -> str: + ) -> SimpleNamespace: train_calls.append(kwargs) - return "trained" + return expected_result monkeypatch.setattr(eval_api, "run_tlmtc_train", run_tlmtc_train) @@ -125,7 +140,7 @@ def run_tlmtc_train( train_kwargs={"batch_size": 16, "run_id": "override-run"}, ) - assert result == "trained" + assert result is expected_result assert len(train_calls) == 1 assert train_calls[0]["labeled_data"] is tlmtc_frame assert {key: value for key, value in train_calls[0].items() if key != "labeled_data"} == { @@ -145,14 +160,16 @@ def test_train_evaluator_resolves_annotation_export_for_selected_task( monkeypatch: pytest.MonkeyPatch, ) -> None: """train_evaluator asks path resolution for the selected export and task.""" - expected_result = object() + expected_result = _train_result("export-run") seen: dict[str, Any] = {} + export_call: dict[str, Any] = {} tlmtc_frame = pd.DataFrame({"text": ["query"], "text_pair": ["answer"], "label_helpful": [1]}) @dataclass(frozen=True, slots=True) class FakeTrainPaths: tool_root: Path training_input_csv: Path + annotation_export_id: str | None def ensure_dirs(self) -> "FakeTrainPaths": seen["ensure_dirs_called"] = True @@ -174,12 +191,14 @@ def resolve_eval_train_paths( return FakeTrainPaths( tool_root=workspace.tool_root("eval"), training_input_csv=tmp_path / "annotation" / "exports" / "export-1" / "generation.csv", + annotation_export_id=export_id, ) monkeypatch.setattr(eval_api, "resolve_eval_train_paths", resolve_eval_train_paths) monkeypatch.setattr(eval_api, "import_eval_train_frame", lambda *, path, task: pd.DataFrame({"path": [path]})) monkeypatch.setattr(eval_api, "build_tlmtc_frame", lambda frame, *, task, mode: tlmtc_frame) monkeypatch.setattr(eval_api, "run_tlmtc_train", lambda **kwargs: expected_result) + monkeypatch.setattr(eval_api, "export_eval_train_meta", lambda **kwargs: export_call.update(kwargs)) result = eval_api.train_evaluator( export_id="export-1", @@ -197,3 +216,8 @@ def resolve_eval_train_paths( }, "ensure_dirs_called": True, } + assert export_call["meta"].model_dump(exclude={"created_at"}) == { + "run_id": "export-run", + "task": Task.GENERATION, + "annotation_export_id": "export-1", + } diff --git a/tests/unit/core/eval/__init__.py b/tests/unit/core/eval/__init__.py new file mode 100644 index 00000000..08ef1f1f --- /dev/null +++ b/tests/unit/core/eval/__init__.py @@ -0,0 +1 @@ +"""Eval unit tests.""" diff --git a/tests/unit/core/eval/test_export.py b/tests/unit/core/eval/test_export.py new file mode 100644 index 00000000..bf52c6cf --- /dev/null +++ b/tests/unit/core/eval/test_export.py @@ -0,0 +1,31 @@ +"""Unit tests for eval export.""" + +import json +from datetime import UTC, datetime +from pathlib import Path + +from pragmata.core.eval.export import export_eval_train_meta +from pragmata.core.schemas.annotation_task import Task +from pragmata.core.schemas.eval_output import EvalTrainMeta + + +def test_export_eval_train_meta_serializes_metadata_json_values( + tmp_path: Path, +) -> None: + """export_eval_train_meta should serialize metadata using JSON-compatible model_dump output.""" + meta = EvalTrainMeta( + run_id="train-run-1", + created_at=datetime(2026, 5, 28, 13, 30, tzinfo=UTC), + task=Task.RETRIEVAL, + annotation_export_id="export-1", + ) + meta_path = tmp_path / "pragmata_train.meta.json" + + export_eval_train_meta(meta=meta, path=meta_path) + + assert json.loads(meta_path.read_text(encoding="utf-8")) == { + "run_id": "train-run-1", + "created_at": "2026-05-28T13:30:00Z", + "task": "retrieval", + "annotation_export_id": "export-1", + } diff --git a/tests/unit/core/paths/test_eval_paths.py b/tests/unit/core/paths/test_eval_paths.py index f564c25b..53a7ac70 100644 --- a/tests/unit/core/paths/test_eval_paths.py +++ b/tests/unit/core/paths/test_eval_paths.py @@ -10,6 +10,7 @@ EvalTrainPaths, find_latest_annotation_export_id, resolve_eval_score_paths, + resolve_eval_train_meta_path, resolve_eval_train_paths, ) from pragmata.core.paths.paths import WorkspacePaths @@ -319,6 +320,18 @@ def test_resolve_eval_train_paths_uses_latest_annotation_export_when_selector_is ) +def test_resolve_eval_train_meta_path_returns_train_run_sidecar_path( + workspace: WorkspacePaths, +) -> None: + """Pragmata train metadata is stored beside the tlmtc train run artifacts.""" + path = resolve_eval_train_meta_path( + workspace=workspace, + run_id="train-run-26", + ) + + assert path == workspace.base_dir / "eval" / "train_outputs" / "train-run-26" / "pragmata_train.meta.json" + + def test_resolve_eval_score_paths_returns_expected_bundle( workspace: WorkspacePaths, ) -> None: diff --git a/tests/unit/core/schemas/test_eval_output.py b/tests/unit/core/schemas/test_eval_output.py index 86955d42..7564f22e 100644 --- a/tests/unit/core/schemas/test_eval_output.py +++ b/tests/unit/core/schemas/test_eval_output.py @@ -7,6 +7,7 @@ from pragmata.core.schemas.annotation_task import Task from pragmata.core.schemas.eval_output import ( + EvalTrainMeta, GenerationScoreReport, GroundingScoreReport, RetrievalScoreReport, @@ -62,6 +63,41 @@ def valid_generation_report(): } +def test_eval_train_meta_accepts_valid_payload() -> None: + """EvalTrainMeta captures the Pragmata-owned run/task link.""" + meta = EvalTrainMeta( + run_id="train-run-1", + created_at=NOW, + task=Task.RETRIEVAL, + annotation_export_id="export-1", + ) + + assert meta.run_id == "train-run-1" + assert meta.created_at == NOW + assert meta.task == Task.RETRIEVAL + assert meta.annotation_export_id == "export-1" + + +def test_eval_train_meta_defaults_created_at_and_export_id() -> None: + """EvalTrainMeta supports standalone training with an internally stamped timestamp.""" + meta = EvalTrainMeta(run_id="train-run-1", task=Task.GROUNDING) + + assert meta.created_at.tzinfo is UTC + assert meta.annotation_export_id is None + + +def test_eval_train_meta_rejects_extra_fields() -> None: + """EvalTrainMeta rejects accidental artifact-shape drift.""" + with pytest.raises(ValidationError): + EvalTrainMeta.model_validate( + { + "run_id": "train-run-1", + "task": "generation", + "label_names": ["helpful"], + } + ) + + def test_retrieval_report_constructs(valid_retrieval_report): """Retrieval score report constructs with retrieval task identity.""" report = RetrievalScoreReport(**valid_retrieval_report)