From b9a1f66626e1cfc140a306053c78f657b5c3c992 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 10:26:14 -0400 Subject: [PATCH 01/14] Add LlamaIndex retrieval instrumentation --- .../README.rst | 33 ++++- .../genai/llama_index/_handler.py | 63 ++++++++- .../tests/conformance/retrieval.py | 65 +++++++++ .../tests/test_conformance.py | 3 +- .../tests/test_retrieval.py | 126 ++++++++++++++++++ 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/README.rst b/instrumentation/opentelemetry-instrumentation-genai-llama-index/README.rst index 73b769c42..05ce39e1f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/README.rst @@ -12,7 +12,8 @@ This package contains OpenTelemetry instrumentation for It emits ``invoke_workflow`` spans for ``AgentWorkflow`` runs, ``invoke_agent`` spans for standalone and workflow-member ``FunctionAgent`` and ``ReActAgent`` executions, and ``execute_tool`` spans when LlamaIndex -executes tools. Model calls +executes tools. It also emits ``retrieval`` spans for synchronous and +asynchronous ``BaseRetriever`` operations. Model calls delegated to provider SDKs are intentionally left to those SDKs' OpenTelemetry instrumentations. @@ -34,6 +35,36 @@ Usage LlamaIndexInstrumentor().instrument() +How instrumentation works +-------------------------- + +The instrumentor registers a LlamaIndex span handler with its dispatcher. The +handler observes LlamaIndex-owned operations and delegates span creation and +completion to ``opentelemetry-util-genai``. Provider model calls are left to +the provider's instrumentation, so they can be composed without duplicate +inference spans. + +.. code-block:: mermaid + + flowchart TD + A[LlamaIndexInstrumentor.instrument] --> B[LlamaIndex dispatcher] + B --> C{Span callback} + C -->|AgentWorkflow.run| D[workflow invocation] + C -->|BaseWorkflowAgent.run / run_agent_step| E[agent invocation] + C -->|call_tool / FunctionTool.call| F[tool invocation] + C -->|BaseRetriever.retrieve / aretrieve| G[retrieval invocation] + D --> H[TelemetryHandler.workflow] + E --> I[TelemetryHandler.invoke_local_agent] + F --> J[TelemetryHandler.tool] + G --> K[TelemetryHandler.retrieval] + H --> L[Start OTel span] + I --> L + J --> L + K --> L + L --> M[Dispatcher exit or error callback] + M --> N[Set attributes and stop/fail invocation] + P[Provider SDK instrumentation] -. model calls .-> Q[inference spans] + Configuration ------------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index c5da051e3..83cfdb2b9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -21,6 +21,7 @@ ToolCall, ToolCallResult, ) +from llama_index.core.base.base_retriever import BaseRetriever from llama_index.core.base.llms.types import ( AudioBlock, ChatMessage, @@ -32,6 +33,7 @@ ) from llama_index.core.instrumentation.span import BaseSpan from llama_index.core.instrumentation.span_handlers import BaseSpanHandler +from llama_index.core.schema import NodeWithScore, QueryBundle from llama_index.core.tools import BaseTool, FunctionTool, ToolOutput from pydantic import PrivateAttr @@ -41,6 +43,7 @@ from opentelemetry.util.genai.invocation import ( AgentInvocation, GenAIInvocation, + RetrievalInvocation, ToolInvocation, WorkflowInvocation, ) @@ -298,6 +301,50 @@ def _request_model(agent: BaseWorkflowAgent) -> str | None: return model_name if isinstance(model_name, str) and model_name else None +def _retrieval_query(bound_args: inspect.BoundArguments) -> str | None: + """Extract text from either accepted LlamaIndex retrieval query form.""" + query = bound_args.arguments.get("str_or_query_bundle") + if isinstance(query, str): + return query + if isinstance(query, QueryBundle): + return query.query_str + return None + + +def _retrieval_top_k(retriever: BaseRetriever) -> int | None: + """Read the common top-k setting without requiring a retriever subtype.""" + try: + top_k = getattr(retriever, "similarity_top_k", None) + except Exception: + return None + if isinstance(top_k, int) and not isinstance(top_k, bool): + return top_k + return None + + +def _retrieval_documents( + result: object, +) -> list[dict[str, Any]] | None: + """Convert retrieved LlamaIndex nodes to semconv document objects.""" + if not isinstance(result, Sequence): + return None + documents: list[dict[str, Any]] = [] + for candidate in cast(Sequence[object], result): + if not isinstance(candidate, NodeWithScore): + continue + try: + document: dict[str, Any] = { + "id": candidate.node_id, + "content": candidate.get_content(), + } + if candidate.score is not None: + document["score"] = candidate.score + documents.append(document) + except Exception: + continue + return documents + + def _tool_attributes( candidate: object, ) -> tuple[str, str, str | None] | None: @@ -750,7 +797,7 @@ def finalize_workflow_agents( class LlamaIndexSpanHandler(BaseSpanHandler[_LlamaIndexInvocation]): - """Map LlamaIndex-owned agent and tool operations to GenAI spans.""" + """Map LlamaIndex-owned agent, tool, and retrieval operations to spans.""" _handler: TelemetryHandler = PrivateAttr() @@ -778,7 +825,7 @@ def new_span( tags: dict[str, Any] | None = None, **kwargs: Any, ) -> _LlamaIndexInvocation | None: - """Start GenAI invocations for LlamaIndex-owned agents and tools. + """Start GenAI invocations for agents, tools, and retrievers. Provider inference is deliberately ignored so its own instrumentation can emit inference telemetry, and nested tool callbacks are deduplicated. @@ -907,6 +954,15 @@ def new_span( if workflow_run_id is not None: parent.register_workflow_agent(workflow_run_id, agent) workflow_agent = agent + elif isinstance(instance, BaseRetriever) and method_name in { + "retrieve", + "aretrieve", + }: + retrieval_invocation = self._handler.retrieval() + retrieval_invocation.top_k = _retrieval_top_k(instance) + if retrieval_invocation.should_capture_content: + retrieval_invocation.query_text = _retrieval_query(bound_args) + invocation = retrieval_invocation elif method_name == "call_tool" and isinstance( (tool_call := bound_args.arguments.get("ev")), ToolCall ): @@ -1132,6 +1188,9 @@ def prepare_to_exit_span( if not _agent_step_is_complete(result): self._expect_workflow_tools(span, result) return span + elif isinstance(span._invocation, RetrievalInvocation): + if span._invocation.should_capture_content: + span._invocation.documents = _retrieval_documents(result) elif isinstance(span._invocation, ToolInvocation): span.reset_workflow_tool() tool_output: ToolOutput | None = None diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py new file mode 100644 index 000000000..3d4606d71 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py @@ -0,0 +1,65 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any + +from llama_index.core.base.base_retriever import BaseRetriever +from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode + +from opentelemetry.instrumentation.genai.llama_index import ( + LlamaIndexInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import ExpectedViolation, Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class _ConformanceRetriever(BaseRetriever): + def __init__(self) -> None: + super().__init__() + self.similarity_top_k = 1 + + def _retrieve(self, query_bundle: QueryBundle) -> list[NodeWithScore]: + return [ + NodeWithScore( + node=TextNode( + id_="capital-france", + text="Paris is the capital of France.", + ), + score=0.99, + ) + ] + + +class RetrievalScenario(Scenario): + expected_spans = {"retrieval": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + LlamaIndexInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + _ConformanceRetriever().retrieve( + "What is the capital of France?" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_conformance.py index c8103fb31..2ec46b42e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_conformance.py @@ -14,12 +14,13 @@ from opentelemetry.test_util_genai.conformance import Scenario, run_conformance from .conformance.agent import AgentScenario +from .conformance.retrieval import RetrievalScenario from .conformance.workflow import WorkflowScenario @pytest.mark.parametrize( "scenario", - [AgentScenario(), WorkflowScenario()], + [AgentScenario(), WorkflowScenario(), RetrievalScenario()], ids=lambda scenario: type(scenario).__name__, ) def test_conformance( diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py new file mode 100644 index 000000000..37868f35c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -0,0 +1,126 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json + +import pytest +from llama_index.core.base.base_retriever import BaseRetriever +from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv.attributes import error_attributes as ErrorAttributes +from opentelemetry.trace import SpanKind, StatusCode + +_GEN_AI_RETRIEVAL_TOP_K = "gen_ai.retrieval.top_k" + + +class _Retriever(BaseRetriever): + def __init__(self, error: BaseException | None = None) -> None: + super().__init__() + self.similarity_top_k = 2 + self._error = error + + def _retrieve(self, query_bundle: QueryBundle) -> list[NodeWithScore]: + if self._error: + raise self._error + return [ + NodeWithScore( + node=TextNode(id_="doc-1", text="Paris is in France."), + score=0.9, + ) + ] + + +def _span(exporter): + spans = [ + s for s in exporter.get_finished_spans() if s.name == "retrieval" + ] + assert len(spans) == 1 + return spans[0] + + +def test_retrieval_captures_documents_and_query( + span_exporter, instrument_llama_index_with_content +) -> None: + _Retriever().retrieve("Where is Paris?") + span = _span(span_exporter) + assert span.kind == SpanKind.CLIENT + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "retrieval" + ) + assert span.attributes[_GEN_AI_RETRIEVAL_TOP_K] == 2 + assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( + "Where is Paris?" + ) + assert json.loads( + span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + ) == [{"id": "doc-1", "content": "Paris is in France.", "score": 0.9}] + + +def test_retrieval_omits_content_without_capture( + span_exporter, instrument_llama_index +) -> None: + _Retriever().retrieve("Where is Paris?") + attrs = _span(span_exporter).attributes + assert GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT not in attrs + assert GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS not in attrs + + +def test_retrieval_uses_snapshotted_capture_setting( + span_exporter, instrument_llama_index_with_content, monkeypatch +) -> None: + # Changing the environment after instrumentation must not alter the + # invocation's content-capture decision. + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "NO_CONTENT" + ) + _Retriever().retrieve("Where is Paris?") + attrs = _span(span_exporter).attributes + assert attrs[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( + "Where is Paris?" + ) + assert GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS in attrs + + +@pytest.mark.asyncio +async def test_async_retrieval_captures_query_and_documents( + span_exporter, instrument_llama_index_with_content +) -> None: + result = await _Retriever().aretrieve("Where is Paris?") + assert result[0].node_id == "doc-1" + span = _span(span_exporter) + assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( + "Where is Paris?" + ) + assert json.loads( + span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + )[0]["content"] == "Paris is in France." + + +def test_sync_retrieval_error_is_unchanged( + span_exporter, instrument_llama_index +) -> None: + error = ValueError("retrieval failed") + with pytest.raises(ValueError) as caught: + _Retriever(error).retrieve("query") + assert caught.value is error + span = _span(span_exporter) + assert span.status.status_code == StatusCode.ERROR + assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" + + +@pytest.mark.asyncio +async def test_async_retrieval_error_is_unchanged( + span_exporter, instrument_llama_index +) -> None: + error = ValueError("retrieval failed") + with pytest.raises(ValueError) as caught: + await _Retriever(error).aretrieve("query") + assert caught.value is error + span = _span(span_exporter) + assert span.status.status_code == StatusCode.ERROR + assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" From 360f9fc95fec9557c5fa69f0057d10b037067c0b Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 10:26:57 -0400 Subject: [PATCH 02/14] Add LlamaIndex retrieval changelog --- .../.changelog/697.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added new file mode 100644 index 000000000..f1f83d878 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added @@ -0,0 +1 @@ +Add retrieval span instrumentation for LlamaIndex ``BaseRetriever`` operations. From 1844ec7a8b6c1755cebedc7911143b56aef71e16 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 10:46:10 -0400 Subject: [PATCH 03/14] Use shared retrieval document model --- .../instrumentation/genai/llama_index/_handler.py | 7 ++++--- util/opentelemetry-util-genai/.changelog/697.added | 1 + .../util/genai/_retrieval_invocation.py | 7 ++++--- .../src/opentelemetry/util/genai/types.py | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 util/opentelemetry-util-genai/.changelog/697.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index 7f1ddee4d..f2542d602 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -55,6 +55,7 @@ MessagePart, OutputMessage, ReasoningPart, + RetrievalDocument, Role, SystemInstructionPart, TextPart, @@ -324,16 +325,16 @@ def _retrieval_top_k(retriever: BaseRetriever) -> int | None: def _retrieval_documents( result: object, -) -> list[dict[str, Any]] | None: +) -> list[RetrievalDocument] | None: """Convert retrieved LlamaIndex nodes to semconv document objects.""" if not isinstance(result, Sequence): return None - documents: list[dict[str, Any]] = [] + documents: list[RetrievalDocument] = [] for candidate in cast(Sequence[object], result): if not isinstance(candidate, NodeWithScore): continue try: - document: dict[str, Any] = { + document: RetrievalDocument = { "id": candidate.node_id, "content": candidate.get_content(), } diff --git a/util/opentelemetry-util-genai/.changelog/697.added b/util/opentelemetry-util-genai/.changelog/697.added new file mode 100644 index 000000000..38afbeb61 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/697.added @@ -0,0 +1 @@ +Add the shared ``RetrievalDocument`` typed model for retrieval telemetry. diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py index 3af2c9245..f88a97a81 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py @@ -3,8 +3,8 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Any, Final +from collections.abc import Sequence +from typing import Final from opentelemetry._logs import Logger from opentelemetry.semconv._incubating.attributes import ( @@ -15,6 +15,7 @@ from opentelemetry.util.genai._instruments import _Instruments from opentelemetry.util.genai._invocation import Error, GenAIInvocation from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.types import RetrievalDocument from opentelemetry.util.genai.utils import ( ContentCapturingMode, gen_ai_json_dumps, @@ -79,7 +80,7 @@ def __init__( self._server_port: int | None = server_port self.top_k: int | None = None self.query_text: str | None = None - self.documents: Sequence[Mapping[str, Any]] | None = None + self.documents: Sequence[RetrievalDocument] | None = None self._start(self._get_start_attributes()) def _get_start_attributes(self) -> dict[str, AttributeValue]: diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py index d041923cd..3aea04ca7 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py @@ -11,6 +11,7 @@ Any, Literal, TypeAlias, + TypedDict, Union, ) @@ -46,6 +47,19 @@ class GenericPart: type: str +class RetrievalDocument(TypedDict, total=False): + """A document returned by a retrieval operation. + + This model follows the GenAI retrieval document schema and is shared by + instrumentations so document attributes retain a consistent shape. + """ + + id: str + score: float + content: str + metadata: dict[str, object] + + @dataclass() class ToolCallRequestPart: """Represents a tool call requested by the model (message part only). From 5e4f1695b42d21dd28be2512f4003c5264bd9a5c Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 10:59:14 -0400 Subject: [PATCH 04/14] Update retrieval changelog fragments --- .../.changelog/{697.added => 698.added} | 0 util/opentelemetry-util-genai/.changelog/{697.added => 698.added} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/{697.added => 698.added} (100%) rename util/opentelemetry-util-genai/.changelog/{697.added => 698.added} (100%) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/698.added similarity index 100% rename from instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/697.added rename to instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/698.added diff --git a/util/opentelemetry-util-genai/.changelog/697.added b/util/opentelemetry-util-genai/.changelog/698.added similarity index 100% rename from util/opentelemetry-util-genai/.changelog/697.added rename to util/opentelemetry-util-genai/.changelog/698.added From 1b1d5b5d51c7a107215df00acfd8d2cc741647e7 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 11:06:27 -0400 Subject: [PATCH 05/14] Refactor import statements and improve code formatting in retrieval tests. --- .../genai/llama_index/_handler.py | 2 +- .../tests/conformance/retrieval.py | 9 +++++---- .../tests/test_retrieval.py | 17 ++++++++++------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index f2542d602..ee30fe843 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -42,8 +42,8 @@ from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( GenAIInvocation, - RetrievalInvocation, LocalAgentInvocation, + RetrievalInvocation, ToolInvocation, WorkflowInvocation, ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py index 3d4606d71..752478776 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/retrieval.py @@ -14,7 +14,10 @@ from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.test_util_genai.conformance import ExpectedViolation, Scenario +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) from opentelemetry.test_util_genai.instrumentor import instrument @@ -60,6 +63,4 @@ def run( meter_provider=meter_provider, content_capture="SPAN_ONLY", ): - _ConformanceRetriever().retrieve( - "What is the capital of France?" - ) + _ConformanceRetriever().retrieve("What is the capital of France?") diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index 37868f35c..e1c207661 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -12,7 +12,9 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) -from opentelemetry.semconv.attributes import error_attributes as ErrorAttributes +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) from opentelemetry.trace import SpanKind, StatusCode _GEN_AI_RETRIEVAL_TOP_K = "gen_ai.retrieval.top_k" @@ -36,9 +38,7 @@ def _retrieve(self, query_bundle: QueryBundle) -> list[NodeWithScore]: def _span(exporter): - spans = [ - s for s in exporter.get_finished_spans() if s.name == "retrieval" - ] + spans = [s for s in exporter.get_finished_spans() if s.name == "retrieval"] assert len(spans) == 1 return spans[0] @@ -96,9 +96,12 @@ async def test_async_retrieval_captures_query_and_documents( assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( "Where is Paris?" ) - assert json.loads( - span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] - )[0]["content"] == "Paris is in France." + assert ( + json.loads( + span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + )[0]["content"] + == "Paris is in France." + ) def test_sync_retrieval_error_is_unchanged( From 2fd996c968b60997e539426f6d3bd76f78cea036 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:00:14 -0400 Subject: [PATCH 06/14] Fix LlamaIndex retrieval document content --- .../instrumentation/genai/llama_index/_handler.py | 2 +- .../tests/test_retrieval.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index ee30fe843..3efc6749c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -336,7 +336,7 @@ def _retrieval_documents( try: document: RetrievalDocument = { "id": candidate.node_id, - "content": candidate.get_content(), + "content": candidate.node.get_content(), } if candidate.score is not None: document["score"] = candidate.score diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index e1c207661..d4dda9ba1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -96,12 +96,9 @@ async def test_async_retrieval_captures_query_and_documents( assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( "Where is Paris?" ) - assert ( - json.loads( - span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] - )[0]["content"] - == "Paris is in France." - ) + assert json.loads( + span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + ) == [{"id": "doc-1", "content": "Paris is in France.", "score": 0.9}] def test_sync_retrieval_error_is_unchanged( From 3fbdab0d978a5f9a653095c1158e3ac5abcf2cce Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:03:17 -0400 Subject: [PATCH 07/14] Test LlamaIndex QueryBundle retrieval --- .../tests/test_retrieval.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index d4dda9ba1..bb13f8f45 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -61,6 +61,16 @@ def test_retrieval_captures_documents_and_query( ) == [{"id": "doc-1", "content": "Paris is in France.", "score": 0.9}] +def test_retrieval_query_bundle_captures_query( + span_exporter, instrument_llama_index_with_content +) -> None: + _Retriever().retrieve(QueryBundle(query_str="Where is Paris?")) + span = _span(span_exporter) + assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( + "Where is Paris?" + ) + + def test_retrieval_omits_content_without_capture( span_exporter, instrument_llama_index ) -> None: From 0689e35c83a2c0f26ec6dc26b151bb056c09c513 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:10:42 -0400 Subject: [PATCH 08/14] Keep retrieval documents instrumentation-local --- .../instrumentation/genai/llama_index/_handler.py | 7 +++---- util/opentelemetry-util-genai/.changelog/698.added | 1 - .../util/genai/_retrieval_invocation.py | 7 +++---- .../src/opentelemetry/util/genai/types.py | 14 -------------- 4 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 util/opentelemetry-util-genai/.changelog/698.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index 3efc6749c..b7ffcb271 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -55,7 +55,6 @@ MessagePart, OutputMessage, ReasoningPart, - RetrievalDocument, Role, SystemInstructionPart, TextPart, @@ -325,16 +324,16 @@ def _retrieval_top_k(retriever: BaseRetriever) -> int | None: def _retrieval_documents( result: object, -) -> list[RetrievalDocument] | None: +) -> list[dict[str, Any]] | None: """Convert retrieved LlamaIndex nodes to semconv document objects.""" if not isinstance(result, Sequence): return None - documents: list[RetrievalDocument] = [] + documents: list[dict[str, Any]] = [] for candidate in cast(Sequence[object], result): if not isinstance(candidate, NodeWithScore): continue try: - document: RetrievalDocument = { + document: dict[str, Any] = { "id": candidate.node_id, "content": candidate.node.get_content(), } diff --git a/util/opentelemetry-util-genai/.changelog/698.added b/util/opentelemetry-util-genai/.changelog/698.added deleted file mode 100644 index 38afbeb61..000000000 --- a/util/opentelemetry-util-genai/.changelog/698.added +++ /dev/null @@ -1 +0,0 @@ -Add the shared ``RetrievalDocument`` typed model for retrieval telemetry. diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py index f88a97a81..3af2c9245 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_retrieval_invocation.py @@ -3,8 +3,8 @@ from __future__ import annotations -from collections.abc import Sequence -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Any, Final from opentelemetry._logs import Logger from opentelemetry.semconv._incubating.attributes import ( @@ -15,7 +15,6 @@ from opentelemetry.util.genai._instruments import _Instruments from opentelemetry.util.genai._invocation import Error, GenAIInvocation from opentelemetry.util.genai.completion_hook import CompletionHook -from opentelemetry.util.genai.types import RetrievalDocument from opentelemetry.util.genai.utils import ( ContentCapturingMode, gen_ai_json_dumps, @@ -80,7 +79,7 @@ def __init__( self._server_port: int | None = server_port self.top_k: int | None = None self.query_text: str | None = None - self.documents: Sequence[RetrievalDocument] | None = None + self.documents: Sequence[Mapping[str, Any]] | None = None self._start(self._get_start_attributes()) def _get_start_attributes(self) -> dict[str, AttributeValue]: diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py index 3aea04ca7..d041923cd 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py @@ -11,7 +11,6 @@ Any, Literal, TypeAlias, - TypedDict, Union, ) @@ -47,19 +46,6 @@ class GenericPart: type: str -class RetrievalDocument(TypedDict, total=False): - """A document returned by a retrieval operation. - - This model follows the GenAI retrieval document schema and is shared by - instrumentations so document attributes retain a consistent shape. - """ - - id: str - score: float - content: str - metadata: dict[str, object] - - @dataclass() class ToolCallRequestPart: """Represents a tool call requested by the model (message part only). From a5d5bfb0b3cc25c272566b82192f8d063ca02a41 Mon Sep 17 00:00:00 2001 From: Surya Date: Sat, 12 Sep 2026 15:21:07 -0400 Subject: [PATCH 09/14] Change exception handling to BaseException Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../opentelemetry/instrumentation/genai/llama_index/_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index b7ffcb271..1ed2afa7d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -340,7 +340,7 @@ def _retrieval_documents( if candidate.score is not None: document["score"] = candidate.score documents.append(document) - except Exception: + except BaseException: continue return documents From 3343682b3de0e975ba14d2958aa71710b1792b95 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:25:51 -0400 Subject: [PATCH 10/14] Harden retrieval metadata extraction --- .../genai/llama_index/_handler.py | 8 ++++++-- .../tests/test_retrieval.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index 1ed2afa7d..38af82526 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -315,7 +315,7 @@ def _retrieval_top_k(retriever: BaseRetriever) -> int | None: """Read the common top-k setting without requiring a retriever subtype.""" try: top_k = getattr(retriever, "similarity_top_k", None) - except Exception: + except BaseException: return None if isinstance(top_k, int) and not isinstance(top_k, bool): return top_k @@ -342,7 +342,11 @@ def _retrieval_documents( documents.append(document) except BaseException: continue - return documents + # Preserve [] for a genuine empty result, but omit the attribute when a + # non-empty result could not be converted into semantic-convention docs. + if documents: + return documents + return [] if len(result) == 0 else None def _tool_attributes( diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index bb13f8f45..55030c96f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -4,11 +4,16 @@ from __future__ import annotations import json +from typing import cast import pytest from llama_index.core.base.base_retriever import BaseRetriever from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode +from opentelemetry.instrumentation.genai.llama_index._handler import ( + _retrieval_documents, + _retrieval_top_k, +) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) @@ -20,6 +25,20 @@ _GEN_AI_RETRIEVAL_TOP_K = "gen_ai.retrieval.top_k" +def test_unconvertible_retrieval_results_are_omitted() -> None: + assert _retrieval_documents([]) == [] + assert _retrieval_documents([object()]) is None + + +def test_similarity_top_k_getter_failure_is_ignored() -> None: + class _FailingRetriever: + @property + def similarity_top_k(self) -> int: + raise KeyboardInterrupt + + assert _retrieval_top_k(cast(BaseRetriever, _FailingRetriever())) is None + + class _Retriever(BaseRetriever): def __init__(self, error: BaseException | None = None) -> None: super().__init__() From b35eb139e0729787f0223637ca437e2b83ee4947 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:29:45 -0400 Subject: [PATCH 11/14] Assert retrieval attribute types --- .../tests/test_retrieval.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index 55030c96f..4522d0181 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -71,13 +71,17 @@ def test_retrieval_captures_documents_and_query( assert ( span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "retrieval" ) - assert span.attributes[_GEN_AI_RETRIEVAL_TOP_K] == 2 - assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( - "Where is Paris?" - ) - assert json.loads( - span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] - ) == [{"id": "doc-1", "content": "Paris is in France.", "score": 0.9}] + top_k = span.attributes[_GEN_AI_RETRIEVAL_TOP_K] + query_text = span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] + documents = span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + assert type(top_k) is int + assert type(query_text) is str + assert type(documents) is str + assert top_k == 2 + assert query_text == "Where is Paris?" + assert json.loads(documents) == [ + {"id": "doc-1", "content": "Paris is in France.", "score": 0.9} + ] def test_retrieval_query_bundle_captures_query( @@ -122,12 +126,14 @@ async def test_async_retrieval_captures_query_and_documents( result = await _Retriever().aretrieve("Where is Paris?") assert result[0].node_id == "doc-1" span = _span(span_exporter) - assert span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] == ( - "Where is Paris?" - ) - assert json.loads( - span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] - ) == [{"id": "doc-1", "content": "Paris is in France.", "score": 0.9}] + query_text = span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_QUERY_TEXT] + documents = span.attributes[GenAIAttributes.GEN_AI_RETRIEVAL_DOCUMENTS] + assert type(query_text) is str + assert type(documents) is str + assert query_text == "Where is Paris?" + assert json.loads(documents) == [ + {"id": "doc-1", "content": "Paris is in France.", "score": 0.9} + ] def test_sync_retrieval_error_is_unchanged( From 29c1b86ace8d51a01686ba120841b90fdcd838ed Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sat, 12 Sep 2026 15:47:36 -0400 Subject: [PATCH 12/14] Refactor _retrieval_documents to improve type casting and return logic --- .../instrumentation/genai/llama_index/_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index 38af82526..ac28388f5 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -328,8 +328,9 @@ def _retrieval_documents( """Convert retrieved LlamaIndex nodes to semconv document objects.""" if not isinstance(result, Sequence): return None + candidates = cast(Sequence[object], result) documents: list[dict[str, Any]] = [] - for candidate in cast(Sequence[object], result): + for candidate in candidates: if not isinstance(candidate, NodeWithScore): continue try: @@ -346,7 +347,7 @@ def _retrieval_documents( # non-empty result could not be converted into semantic-convention docs. if documents: return documents - return [] if len(result) == 0 else None + return [] if len(candidates) == 0 else None def _tool_attributes( From 17a95d102adbfddd5fc62635bf6256883e9bf315 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sun, 13 Sep 2026 16:11:31 -0400 Subject: [PATCH 13/14] Match retrieval metadata access pattern --- .../instrumentation/genai/llama_index/_handler.py | 5 +---- .../tests/test_retrieval.py | 12 ------------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index ac28388f5..8698a54d5 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -313,10 +313,7 @@ def _retrieval_query(bound_args: inspect.BoundArguments) -> str | None: def _retrieval_top_k(retriever: BaseRetriever) -> int | None: """Read the common top-k setting without requiring a retriever subtype.""" - try: - top_k = getattr(retriever, "similarity_top_k", None) - except BaseException: - return None + top_k = getattr(retriever, "similarity_top_k", None) if isinstance(top_k, int) and not isinstance(top_k, bool): return top_k return None diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index 4522d0181..907b14e16 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -4,15 +4,12 @@ from __future__ import annotations import json -from typing import cast - import pytest from llama_index.core.base.base_retriever import BaseRetriever from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode from opentelemetry.instrumentation.genai.llama_index._handler import ( _retrieval_documents, - _retrieval_top_k, ) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, @@ -30,15 +27,6 @@ def test_unconvertible_retrieval_results_are_omitted() -> None: assert _retrieval_documents([object()]) is None -def test_similarity_top_k_getter_failure_is_ignored() -> None: - class _FailingRetriever: - @property - def similarity_top_k(self) -> int: - raise KeyboardInterrupt - - assert _retrieval_top_k(cast(BaseRetriever, _FailingRetriever())) is None - - class _Retriever(BaseRetriever): def __init__(self, error: BaseException | None = None) -> None: super().__init__() From d9c5f9307ece4b99d8f239dba1e741ac445a26b8 Mon Sep 17 00:00:00 2001 From: eternalcuriouslearner Date: Sun, 13 Sep 2026 16:35:19 -0400 Subject: [PATCH 14/14] Add missing import for json in test_retrieval.py --- .../tests/test_retrieval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py index 907b14e16..65687f25f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/test_retrieval.py @@ -4,6 +4,7 @@ from __future__ import annotations import json + import pytest from llama_index.core.base.base_retriever import BaseRetriever from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode