diff --git a/README.md b/README.md index e4efc38..6dcdc22 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,9 @@ agenteval compare RUN_A1,RUN_A2 vs RUN_B1,RUN_B2 # Multi-run comparison ```bash agenteval import --from agentlens --db sessions.db --output suite.yaml [--grader contains] [--limit 100] + +# OTel GenAI traces → eval fixtures/trajectories (one case per trace; tools → tool-check) +agenteval import --from otel --file traces.json --output suite.yaml [--grader exact] [--limit 100] ``` --- diff --git a/src/agenteval/commands/importers.py b/src/agenteval/commands/importers.py index 5069d58..f419203 100644 --- a/src/agenteval/commands/importers.py +++ b/src/agenteval/commands/importers.py @@ -12,37 +12,54 @@ def register(cli: click.Group, helpers: dict) -> None: """Register the import commands on the CLI group.""" @cli.command("import") - @click.option("--from", "source", required=True, type=click.Choice(["agentlens"]), help="Import source.") - @click.option("--db", required=True, type=click.Path(), help="Path to source database.") + @click.option("--from", "source", required=True, type=click.Choice(["agentlens", "otel"]), help="Import source.") + @click.option("--db", default=None, type=click.Path(), help="Path to source database (agentlens).") + @click.option("--file", "file_path", default=None, type=click.Path(exists=True), help="OTLP traces JSON file (otel).") @click.option("--output", "-o", required=True, type=click.Path(), help="Output YAML suite path.") @click.option("--name", default=None, help="Suite name (defaults to source name).") @click.option("--grader", default="contains", show_default=True, help="Default grader for imported cases.") - @click.option("--limit", default=None, type=int, help="Max sessions to import.") - def import_cmd(source: str, db: str, output: str, name: Optional[str], grader: str, limit: Optional[int]) -> None: - """Import agent sessions from external sources as eval suites. + @click.option("--limit", default=None, type=int, help="Max sessions/traces to import.") + def import_cmd(source, db, file_path, output, name, grader, limit) -> None: + """Import agent sessions / traces from external sources as eval suites. Examples: agenteval import --from agentlens --db sessions.db --output suite.yaml - agenteval import --from agentlens --db sessions.db --output suite.yaml --grader exact --limit 100 + agenteval import --from otel --file traces.json --output suite.yaml --grader exact """ + from agenteval.importers.agentlens import export_suite_yaml + if source == "agentlens": from agenteval.importers.agentlens import ( AgentLensImportError, - export_suite_yaml, import_agentlens, ) + if not db: + click.echo("--db is required for --from agentlens", err=True) + sys.exit(1) suite_name = name or "agentlens-import" try: suite = import_agentlens(db_path=db, suite_name=suite_name, grader=grader, limit=limit) except AgentLensImportError as e: click.echo(f"Import error: {e}", err=True) sys.exit(1) + else: # otel + from agenteval.importers.otel import OtelImportError, import_otel - out_path = export_suite_yaml(suite, output) - click.echo(f"Imported {len(suite.cases)} cases \u2192 {out_path}") + if not file_path: + click.echo("--file is required for --from otel", err=True) + sys.exit(1) + suite_name = name or "otel-import" + try: + suite = import_otel(file_path=file_path, suite_name=suite_name, grader=grader, limit=limit) + except OtelImportError as e: + click.echo(f"Import error: {e}", err=True) + sys.exit(1) + + out_path = export_suite_yaml(suite, output) + click.echo(f"Imported {len(suite.cases)} cases \u2192 {out_path}") @cli.command("import-agentlens") @click.option("--session", default=None, help="Single session ID to import.") diff --git a/src/agenteval/importers/otel.py b/src/agenteval/importers/otel.py new file mode 100644 index 0000000..c1b3c73 --- /dev/null +++ b/src/agenteval/importers/otel.py @@ -0,0 +1,162 @@ +"""Import OTel GenAI traces as eval fixtures / trajectories (#15). + +Reads an OTLP trace export (JSON: resourceSpans → scopeSpans → spans) and turns +each trace into an EvalCase — input from the GenAI prompt, expected output from +the completion, and the ordered tool calls as a trajectory. This is fixture +*ingestion*, NOT a trace dashboard (an explicit anti-goal): the output is a +replayable eval suite, nothing more. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any, Dict, List, Optional + +from agenteval.models import EvalCase, EvalSuite + + +class OtelImportError(Exception): + """Raised when an OTLP trace file can't be read or parsed.""" + + +def _attr(attrs: List[Dict[str, Any]], key: str) -> Optional[str]: + """Read a string-valued OTLP attribute (attrs are [{key, value:{stringValue}}]).""" + for kv in attrs: + if kv.get("key") == key: + v = kv.get("value") or {} + sv = v.get("stringValue") + return sv if isinstance(sv, str) else None + return None + + +def _text_of(value: Any) -> str: + """Coerce a message content (str, or a list of content parts) to text.""" + if isinstance(value, str): + return value + if isinstance(value, list): + parts = [p.get("text", "") if isinstance(p, dict) else str(p) for p in value] + return " ".join(p for p in parts if p) + return "" + + +def _messages_content(raw: Optional[str], role: str) -> Optional[str]: + """Last message with `role` from a gen_ai.{input,output}.messages JSON attr.""" + if not raw: + return None + try: + msgs = json.loads(raw) + except (ValueError, TypeError): + return None + if not isinstance(msgs, list): + return None + found = [_text_of(m.get("content")) for m in msgs if isinstance(m, dict) and m.get("role") == role] + found = [t for t in found if t.strip()] + return found[-1] if found else None + + +def _indexed(attrs: List[Dict[str, Any]], kind: str, role: str) -> Optional[str]: + """OpenLLMetry indexed style: gen_ai.{prompt,completion}.{i}.content for a role.""" + out: List[str] = [] + for i in range(64): + r = _attr(attrs, f"gen_ai.{kind}.{i}.role") + c = _attr(attrs, f"gen_ai.{kind}.{i}.content") + if r is None and c is None: + break + if c and (r == role or r is None): + out.append(c) + return out[-1] if out else None + + +def _extract_prompt(attrs: List[Dict[str, Any]]) -> Optional[str]: + return ( + _messages_content(_attr(attrs, "gen_ai.input.messages"), "user") + or _indexed(attrs, "prompt", "user") + or _attr(attrs, "gen_ai.prompt") + ) + + +def _extract_output(attrs: List[Dict[str, Any]]) -> Optional[str]: + return ( + _messages_content(_attr(attrs, "gen_ai.output.messages"), "assistant") + or _indexed(attrs, "completion", "assistant") + or _attr(attrs, "gen_ai.completion") + ) + + +def _trace_to_case(trace_id: str, spans: List[Dict[str, Any]], grader: str) -> Optional[EvalCase]: + model: Optional[str] = None + input_text: Optional[str] = None + output_text: Optional[str] = None + tools: List[str] = [] + + for span in spans: + attrs = span.get("attributes") or [] + model = _attr(attrs, "gen_ai.request.model") or model + if _attr(attrs, "gen_ai.operation.name") == "execute_tool" or _attr(attrs, "gen_ai.tool.name"): + tools.append(_attr(attrs, "gen_ai.tool.name") or span.get("name") or "tool") + if input_text is None: + input_text = _extract_prompt(attrs) + out = _extract_output(attrs) + if out: + output_text = out # last completion in the trace wins + + if not input_text or not input_text.strip(): + return None + + expected: Dict[str, Any] = {} + if output_text and output_text.strip(): + expected["output"] = output_text.strip() + if tools: + expected["tools"] = tools + + # A trace with tool calls is a trajectory; otherwise a plain replay fixture. + effective_grader = "tool-check" if (tools and grader == "contains") else grader + tags = ["imported", "otel"] + ([model] if model else []) + return EvalCase( + name=f"otel_{(trace_id or 'trace')[:8]}", + input=input_text.strip(), + expected=expected, + grader=effective_grader, + tags=tags, + ) + + +def _all_spans(data: Dict[str, Any]) -> List[Dict[str, Any]]: + spans: List[Dict[str, Any]] = [] + for rs in data.get("resourceSpans") or []: + for ss in rs.get("scopeSpans") or []: + spans.extend(ss.get("spans") or []) + return spans + + +def import_otel( + *, + file_path: str, + suite_name: str = "otel-import", + grader: str = "contains", + limit: Optional[int] = None, +) -> EvalSuite: + """Load an OTLP traces JSON file and build an EvalSuite (one case per trace).""" + try: + with open(file_path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, ValueError) as e: + raise OtelImportError(f"reading {file_path}: {e}") from e + if not isinstance(data, dict): + raise OtelImportError("OTLP trace file must be a JSON object") + + by_trace: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for span in _all_spans(data): + by_trace[str(span.get("traceId", ""))].append(span) + + cases: List[EvalCase] = [] + for trace_id, spans in by_trace.items(): + spans.sort(key=lambda s: int(s.get("startTimeUnixNano") or 0)) + case = _trace_to_case(trace_id, spans, grader) + if case: + cases.append(case) + if limit and len(cases) >= limit: + break + + return EvalSuite(name=suite_name, agent="otel-import", cases=cases) diff --git a/tests/test_otel_import.py b/tests/test_otel_import.py new file mode 100644 index 0000000..29c5920 --- /dev/null +++ b/tests/test_otel_import.py @@ -0,0 +1,89 @@ +"""OTel GenAI trace → eval fixtures/trajectories (#15).""" + +import json + +import pytest + +from agenteval.importers.otel import OtelImportError, _trace_to_case, import_otel + + +def _attr(key, val): + return {"key": key, "value": {"stringValue": val}} + + +def _span(trace_id, name, attrs, start=1): + return {"traceId": trace_id, "name": name, "startTimeUnixNano": str(start), "attributes": attrs} + + +def _llm_span(trace_id, model="gpt-4o", user="What is 2+2?", assistant="4", start=1): + return _span(trace_id, "chat", [ + _attr("gen_ai.request.model", model), + _attr("gen_ai.input.messages", json.dumps([{"role": "user", "content": user}])), + _attr("gen_ai.output.messages", json.dumps([{"role": "assistant", "content": assistant}])), + ], start) + + +def _tool_span(trace_id, tool="calculator", start=2): + return _span(trace_id, "tool", [ + _attr("gen_ai.operation.name", "execute_tool"), + _attr("gen_ai.tool.name", tool), + ], start) + + +def test_trace_to_case_replay_fixture(): + case = _trace_to_case("trace123", [_llm_span("trace123")], "contains") + assert case is not None + assert case.input == "What is 2+2?" + assert case.expected["output"] == "4" + assert case.grader == "contains" + assert "otel" in case.tags and "gpt-4o" in case.tags + + +def test_trace_with_tools_becomes_trajectory(): + spans = [_llm_span("t1", start=1), _tool_span("t1", "calculator", start=2), _tool_span("t1", "search", start=3)] + case = _trace_to_case("t1", spans, "contains") + assert case.expected["tools"] == ["calculator", "search"] + assert case.grader == "tool-check" # contains + tools → tool-check + + +def test_indexed_openllmetry_style(): + span = _span("t2", "chat", [ + _attr("gen_ai.prompt.0.role", "user"), + _attr("gen_ai.prompt.0.content", "hello"), + _attr("gen_ai.completion.0.role", "assistant"), + _attr("gen_ai.completion.0.content", "hi there"), + ]) + case = _trace_to_case("t2", [span], "contains") + assert case.input == "hello" + assert case.expected["output"] == "hi there" + + +def test_span_without_input_is_skipped(): + span = _span("t3", "noop", [_attr("gen_ai.request.model", "gpt-4o")]) + assert _trace_to_case("t3", [span], "contains") is None + + +def test_import_otel_groups_by_trace(tmp_path): + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [ + _llm_span("traceA", user="Q-A", assistant="A-A"), + _llm_span("traceB", user="Q-B", assistant="A-B"), + _tool_span("traceB", "search"), + ]}]} + ] + } + f = tmp_path / "traces.json" + f.write_text(json.dumps(payload), encoding="utf-8") + suite = import_otel(file_path=str(f), suite_name="s") + assert suite.name == "s" + assert len(suite.cases) == 2 # one per trace + by_input = {c.input: c for c in suite.cases} + assert by_input["Q-B"].expected["tools"] == ["search"] + + +def test_import_otel_bad_file_raises(tmp_path): + f = tmp_path / "bad.json" + f.write_text("not json", encoding="utf-8") + with pytest.raises(OtelImportError): + import_otel(file_path=str(f))