feat(import): OTel GenAI traces → eval fixtures/trajectories (#15)

README.md

@@ -271,6 +271,9 @@ agenteval compare RUN_A1,RUN_A2 vs RUN_B1,RUN_B2   # Multi-run comparison
 
 ```bash
 agenteval import --from agentlens --db sessions.db --output suite.yaml [--grader contains] [--limit 100]
+
+# OTel GenAI traces → eval fixtures/trajectories (one case per trace; tools → tool-check)
+agenteval import --from otel --file traces.json --output suite.yaml [--grader exact] [--limit 100]
 ```
 
 ---

src/agenteval/commands/importers.py

@@ -12,37 +12,54 @@ def register(cli: click.Group, helpers: dict) -> None:
     """Register the import commands on the CLI group."""
 
     @cli.command("import")
-    @click.option("--from", "source", required=True, type=click.Choice(["agentlens"]), help="Import source.")
-    @click.option("--db", required=True, type=click.Path(), help="Path to source database.")
+    @click.option("--from", "source", required=True, type=click.Choice(["agentlens", "otel"]), help="Import source.")
+    @click.option("--db", default=None, type=click.Path(), help="Path to source database (agentlens).")
+    @click.option("--file", "file_path", default=None, type=click.Path(exists=True), help="OTLP traces JSON file (otel).")
     @click.option("--output", "-o", required=True, type=click.Path(), help="Output YAML suite path.")
     @click.option("--name", default=None, help="Suite name (defaults to source name).")
     @click.option("--grader", default="contains", show_default=True, help="Default grader for imported cases.")
-    @click.option("--limit", default=None, type=int, help="Max sessions to import.")
-    def import_cmd(source: str, db: str, output: str, name: Optional[str], grader: str, limit: Optional[int]) -> None:
-        """Import agent sessions from external sources as eval suites.
+    @click.option("--limit", default=None, type=int, help="Max sessions/traces to import.")
+    def import_cmd(source, db, file_path, output, name, grader, limit) -> None:
+        """Import agent sessions / traces from external sources as eval suites.
 
         Examples:
 
           agenteval import --from agentlens --db sessions.db --output suite.yaml
 
-          agenteval import --from agentlens --db sessions.db --output suite.yaml --grader exact --limit 100
+          agenteval import --from otel --file traces.json --output suite.yaml --grader exact
         """
+        from agenteval.importers.agentlens import export_suite_yaml
+
         if source == "agentlens":
             from agenteval.importers.agentlens import (
                 AgentLensImportError,
-                export_suite_yaml,
                 import_agentlens,
             )
 
+            if not db:
+                click.echo("--db is required for --from agentlens", err=True)
+                sys.exit(1)
             suite_name = name or "agentlens-import"
             try:
                 suite = import_agentlens(db_path=db, suite_name=suite_name, grader=grader, limit=limit)
             except AgentLensImportError as e:
                 click.echo(f"Import error: {e}", err=True)
                 sys.exit(1)
+        else:  # otel
+            from agenteval.importers.otel import OtelImportError, import_otel
 
-            out_path = export_suite_yaml(suite, output)
-            click.echo(f"Imported {len(suite.cases)} cases \u2192 {out_path}")
+            if not file_path:
+                click.echo("--file is required for --from otel", err=True)
+                sys.exit(1)
+            suite_name = name or "otel-import"
+            try:
+                suite = import_otel(file_path=file_path, suite_name=suite_name, grader=grader, limit=limit)
+            except OtelImportError as e:
+                click.echo(f"Import error: {e}", err=True)
+                sys.exit(1)
+
+        out_path = export_suite_yaml(suite, output)
+        click.echo(f"Imported {len(suite.cases)} cases \u2192 {out_path}")
 
     @cli.command("import-agentlens")
     @click.option("--session", default=None, help="Single session ID to import.")

src/agenteval/importers/otel.py

@@ -0,0 +1,162 @@
+"""Import OTel GenAI traces as eval fixtures / trajectories (#15).
+
+Reads an OTLP trace export (JSON: resourceSpans → scopeSpans → spans) and turns
+each trace into an EvalCase — input from the GenAI prompt, expected output from
+the completion, and the ordered tool calls as a trajectory. This is fixture
+*ingestion*, NOT a trace dashboard (an explicit anti-goal): the output is a
+replayable eval suite, nothing more.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+from agenteval.models import EvalCase, EvalSuite
+
+
+class OtelImportError(Exception):
+    """Raised when an OTLP trace file can't be read or parsed."""
+
+
+def _attr(attrs: List[Dict[str, Any]], key: str) -> Optional[str]:
+    """Read a string-valued OTLP attribute (attrs are [{key, value:{stringValue}}])."""
+    for kv in attrs:
+        if kv.get("key") == key:
+            v = kv.get("value") or {}
+            sv = v.get("stringValue")
+            return sv if isinstance(sv, str) else None
+    return None
+
+
+def _text_of(value: Any) -> str:
+    """Coerce a message content (str, or a list of content parts) to text."""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        parts = [p.get("text", "") if isinstance(p, dict) else str(p) for p in value]
+        return " ".join(p for p in parts if p)
+    return ""
+
+
+def _messages_content(raw: Optional[str], role: str) -> Optional[str]:
+    """Last message with `role` from a gen_ai.{input,output}.messages JSON attr."""
+    if not raw:
+        return None
+    try:
+        msgs = json.loads(raw)
+    except (ValueError, TypeError):
+        return None
+    if not isinstance(msgs, list):
+        return None
+    found = [_text_of(m.get("content")) for m in msgs if isinstance(m, dict) and m.get("role") == role]
+    found = [t for t in found if t.strip()]
+    return found[-1] if found else None
+
+
+def _indexed(attrs: List[Dict[str, Any]], kind: str, role: str) -> Optional[str]:
+    """OpenLLMetry indexed style: gen_ai.{prompt,completion}.{i}.content for a role."""
+    out: List[str] = []
+    for i in range(64):
+        r = _attr(attrs, f"gen_ai.{kind}.{i}.role")
+        c = _attr(attrs, f"gen_ai.{kind}.{i}.content")
+        if r is None and c is None:
+            break
+        if c and (r == role or r is None):
+            out.append(c)
+    return out[-1] if out else None
+
+
+def _extract_prompt(attrs: List[Dict[str, Any]]) -> Optional[str]:
+    return (
+        _messages_content(_attr(attrs, "gen_ai.input.messages"), "user")
+        or _indexed(attrs, "prompt", "user")
+        or _attr(attrs, "gen_ai.prompt")
+    )
+
+
+def _extract_output(attrs: List[Dict[str, Any]]) -> Optional[str]:
+    return (
+        _messages_content(_attr(attrs, "gen_ai.output.messages"), "assistant")
+        or _indexed(attrs, "completion", "assistant")
+        or _attr(attrs, "gen_ai.completion")
+    )
+
+
+def _trace_to_case(trace_id: str, spans: List[Dict[str, Any]], grader: str) -> Optional[EvalCase]:
+    model: Optional[str] = None
+    input_text: Optional[str] = None
+    output_text: Optional[str] = None
+    tools: List[str] = []
+
+    for span in spans:
+        attrs = span.get("attributes") or []
+        model = _attr(attrs, "gen_ai.request.model") or model
+        if _attr(attrs, "gen_ai.operation.name") == "execute_tool" or _attr(attrs, "gen_ai.tool.name"):
+            tools.append(_attr(attrs, "gen_ai.tool.name") or span.get("name") or "tool")
+        if input_text is None:
+            input_text = _extract_prompt(attrs)
+        out = _extract_output(attrs)
+        if out:
+            output_text = out  # last completion in the trace wins
+
+    if not input_text or not input_text.strip():
+        return None
+
+    expected: Dict[str, Any] = {}
+    if output_text and output_text.strip():
+        expected["output"] = output_text.strip()
+    if tools:
+        expected["tools"] = tools
+
+    # A trace with tool calls is a trajectory; otherwise a plain replay fixture.
+    effective_grader = "tool-check" if (tools and grader == "contains") else grader
+    tags = ["imported", "otel"] + ([model] if model else [])
+    return EvalCase(
+        name=f"otel_{(trace_id or 'trace')[:8]}",
+        input=input_text.strip(),
+        expected=expected,
+        grader=effective_grader,
+        tags=tags,
+    )
+
+
+def _all_spans(data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    spans: List[Dict[str, Any]] = []
+    for rs in data.get("resourceSpans") or []:
+        for ss in rs.get("scopeSpans") or []:
+            spans.extend(ss.get("spans") or [])
+    return spans
+
+
+def import_otel(
+    *,
+    file_path: str,
+    suite_name: str = "otel-import",
+    grader: str = "contains",
+    limit: Optional[int] = None,
+) -> EvalSuite:
+    """Load an OTLP traces JSON file and build an EvalSuite (one case per trace)."""
+    try:
+        with open(file_path, encoding="utf-8") as f:
+            data = json.load(f)
+    except (OSError, ValueError) as e:
+        raise OtelImportError(f"reading {file_path}: {e}") from e
+    if not isinstance(data, dict):
+        raise OtelImportError("OTLP trace file must be a JSON object")
+
+    by_trace: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+    for span in _all_spans(data):
+        by_trace[str(span.get("traceId", ""))].append(span)
+
+    cases: List[EvalCase] = []
+    for trace_id, spans in by_trace.items():
+        spans.sort(key=lambda s: int(s.get("startTimeUnixNano") or 0))
+        case = _trace_to_case(trace_id, spans, grader)
+        if case:
+            cases.append(case)
+        if limit and len(cases) >= limit:
+            break
+
+    return EvalSuite(name=suite_name, agent="otel-import", cases=cases)

tests/test_otel_import.py

@@ -0,0 +1,89 @@
+"""OTel GenAI trace → eval fixtures/trajectories (#15)."""
+
+import json
+
+import pytest
+
+from agenteval.importers.otel import OtelImportError, _trace_to_case, import_otel
+
+
+def _attr(key, val):
+    return {"key": key, "value": {"stringValue": val}}
+
+
+def _span(trace_id, name, attrs, start=1):
+    return {"traceId": trace_id, "name": name, "startTimeUnixNano": str(start), "attributes": attrs}
+
+
+def _llm_span(trace_id, model="gpt-4o", user="What is 2+2?", assistant="4", start=1):
+    return _span(trace_id, "chat", [
+        _attr("gen_ai.request.model", model),
+        _attr("gen_ai.input.messages", json.dumps([{"role": "user", "content": user}])),
+        _attr("gen_ai.output.messages", json.dumps([{"role": "assistant", "content": assistant}])),
+    ], start)
+
+
+def _tool_span(trace_id, tool="calculator", start=2):
+    return _span(trace_id, "tool", [
+        _attr("gen_ai.operation.name", "execute_tool"),
+        _attr("gen_ai.tool.name", tool),
+    ], start)
+
+
+def test_trace_to_case_replay_fixture():
+    case = _trace_to_case("trace123", [_llm_span("trace123")], "contains")
+    assert case is not None
+    assert case.input == "What is 2+2?"
+    assert case.expected["output"] == "4"
+    assert case.grader == "contains"
+    assert "otel" in case.tags and "gpt-4o" in case.tags
+
+
+def test_trace_with_tools_becomes_trajectory():
+    spans = [_llm_span("t1", start=1), _tool_span("t1", "calculator", start=2), _tool_span("t1", "search", start=3)]
+    case = _trace_to_case("t1", spans, "contains")
+    assert case.expected["tools"] == ["calculator", "search"]
+    assert case.grader == "tool-check"  # contains + tools → tool-check
+
+
+def test_indexed_openllmetry_style():
+    span = _span("t2", "chat", [
+        _attr("gen_ai.prompt.0.role", "user"),
+        _attr("gen_ai.prompt.0.content", "hello"),
+        _attr("gen_ai.completion.0.role", "assistant"),
+        _attr("gen_ai.completion.0.content", "hi there"),
+    ])
+    case = _trace_to_case("t2", [span], "contains")
+    assert case.input == "hello"
+    assert case.expected["output"] == "hi there"
+
+
+def test_span_without_input_is_skipped():
+    span = _span("t3", "noop", [_attr("gen_ai.request.model", "gpt-4o")])
+    assert _trace_to_case("t3", [span], "contains") is None
+
+
+def test_import_otel_groups_by_trace(tmp_path):
+    payload = {
+        "resourceSpans": [
+            {"scopeSpans": [{"spans": [
+                _llm_span("traceA", user="Q-A", assistant="A-A"),
+                _llm_span("traceB", user="Q-B", assistant="A-B"),
+                _tool_span("traceB", "search"),
+            ]}]}
+        ]
+    }
+    f = tmp_path / "traces.json"
+    f.write_text(json.dumps(payload), encoding="utf-8")
+    suite = import_otel(file_path=str(f), suite_name="s")
+    assert suite.name == "s"
+    assert len(suite.cases) == 2  # one per trace
+    by_input = {c.input: c for c in suite.cases}
+    assert by_input["Q-B"].expected["tools"] == ["search"]
+
+
+def test_import_otel_bad_file_raises(tmp_path):
+    f = tmp_path / "bad.json"
+    f.write_text("not json", encoding="utf-8")
+    with pytest.raises(OtelImportError):
+        import_otel(file_path=str(f))