From b850a3b36fe6ff02abd689cc0959813783c42147 Mon Sep 17 00:00:00 2001 From: dlowzzxx Date: Thu, 10 Sep 2026 09:38:34 +0200 Subject: [PATCH 1/4] feat(langchain): capture document relevance score on retrieval spans (#584) --- .../genai/langchain/callback_handler.py | 74 ++- .../tests/test_callback_handler.py | 425 ++++++++++++++++++ .../tests/test_retriever.py | 330 +++++++++++++- 3 files changed, 800 insertions(+), 29 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 054e4874c..693bfa423 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import math from collections.abc import Mapping, Sequence from typing import Any, cast from uuid import UUID @@ -80,6 +81,73 @@ def _conversation_id(metadata: dict[str, Any] | None) -> str | None: return None +def _extract_document_score(doc: Any) -> float | int | None: + """Extract relevance score polymorphically from a Document or Mapping. + + Checks doc.score first, then falls back to doc.metadata['score']. + Also defensively supports Mapping/dict documents and duck-typed objects. + """ + score: Any = None + if isinstance(doc, Mapping): + doc_map = cast(Mapping[str, Any], doc) + score = doc_map.get("score") + if score is None: + metadata = doc_map.get("metadata") + if isinstance(metadata, Mapping): + meta_map = cast(Mapping[str, Any], metadata) + score = meta_map.get("score") + elif metadata is not None: + score = getattr(metadata, "score", None) + else: + score = getattr(doc, "score", None) + if score is None: + metadata = getattr(doc, "metadata", None) + if isinstance(metadata, Mapping): + meta_map = cast(Mapping[str, Any], metadata) + score = meta_map.get("score") + elif metadata is not None: + score = getattr(metadata, "score", None) + + if ( + score is not None + and not isinstance(score, bool) + and isinstance(score, (int, float)) + ): + if isinstance(score, float) and not math.isfinite(score): + return None + return score + + return None + + +def _document_to_dict(doc: Any) -> dict[str, Any]: + """Convert a Document, duck-typed document object, or Mapping to a dict. + + Extracts content (checking page_content first, then content), id, + and conditionally score if present and numeric. + """ + if isinstance(doc, Mapping): + doc_map = cast(Mapping[str, Any], doc) + content = doc_map.get("page_content") + if content is None: + content = doc_map.get("content") + doc_id = doc_map.get("id") + else: + content = getattr(doc, "page_content", None) + if content is None: + content = getattr(doc, "content", None) + doc_id = getattr(doc, "id", None) + + doc_dict: dict[str, Any] = { + "content": content, + "id": doc_id, + } + score = _extract_document_score(doc) + if score is not None: + doc_dict["score"] = score + return doc_dict + + class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): """ A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and chains, tools etc,. in future. @@ -715,11 +783,7 @@ def on_retriever_end( if self._telemetry_handler.should_capture_content(): invocation.documents = [ - { - "content": doc.page_content, - "id": doc.id, - } - for doc in documents + _document_to_dict(doc) for doc in documents ] invocation.stop() if not invocation.span.is_recording(): diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index baac28150..8930745e6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -9,6 +9,7 @@ """ import base64 +import math import uuid from unittest import mock @@ -37,6 +38,8 @@ from opentelemetry.instrumentation.genai.langchain.callback_handler import ( OpenTelemetryLangChainCallbackHandler, + _document_to_dict, + _extract_document_score, ) from opentelemetry.instrumentation.genai.langchain.utils import ( _legacy_function_call_request, @@ -1542,6 +1545,428 @@ def test_unknown_run_id_does_not_raise(self): handler, _, _ = _make_handler_with_retrieval() handler.on_retriever_end(documents=[], run_id=_run_id()) + def test_document_score_from_attribute(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDoc: + page_content = "text" + id = "doc-1" + score = 0.85 + metadata = {} + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[DuckDoc()], run_id=run_id) + + assert retrieval_inv.documents[0]["score"] == 0.85 + + def test_document_score_from_metadata(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + doc = Document( + page_content="text", id="doc-2", metadata={"score": 0.92} + ) + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[doc], run_id=run_id) + + assert retrieval_inv.documents[0]["score"] == 0.92 + + def test_document_score_precedence(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDoc: + page_content = "text" + id = "doc-3" + score = 0.9 + metadata = {"score": 0.5} + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[DuckDoc()], run_id=run_id) + + assert retrieval_inv.documents[0]["score"] == 0.9 + + def test_document_score_fallback_to_metadata_when_attr_is_none(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDoc: + page_content = "text" + id = "doc-4" + score = None + metadata = {"score": 0.77} + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[DuckDoc()], run_id=run_id) + + assert retrieval_inv.documents[0]["score"] == 0.77 + + def test_document_score_zero_preserved(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDoc: + page_content = "text" + id = "doc-5" + score = 0.0 + + doc_meta = Document( + page_content="text-meta", id="doc-6", metadata={"score": 0} + ) + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end( + documents=[DuckDoc(), doc_meta], run_id=run_id + ) + + assert "score" in retrieval_inv.documents[0] + assert retrieval_inv.documents[0]["score"] == 0.0 + assert "score" in retrieval_inv.documents[1] + assert retrieval_inv.documents[1]["score"] == 0 + + @pytest.mark.parametrize( + "invalid_score", + ["high", True, False, [1.0], {"val": 1}], + ) + def test_document_score_non_numeric_ignored(self, invalid_score): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + doc = Document(page_content="text", metadata={"score": invalid_score}) + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[doc], run_id=run_id) + + assert "score" not in retrieval_inv.documents[0] + + @pytest.mark.parametrize( + "non_finite_score", + [ + float("nan"), + float("inf"), + float("-inf"), + math.nan, + math.inf, + -math.inf, + ], + ) + def test_document_score_non_finite_ignored(self, non_finite_score): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDocAttr: + page_content = "attr text" + score = non_finite_score + + doc_meta = Document( + page_content="meta text", metadata={"score": non_finite_score} + ) + doc_dict = {"page_content": "dict text", "score": non_finite_score} + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end( + documents=[DuckDocAttr(), doc_meta, doc_dict], run_id=run_id + ) + + for item in retrieval_inv.documents: + assert "score" not in item + + def test_document_score_plain_dict_and_duck_typed(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + dict_doc_1 = { + "page_content": "dict content 1", + "id": "dict-1", + "score": 0.88, + } + dict_doc_2 = { + "content": "dict content 2", + "metadata": {"score": 0.72}, + } + dict_doc_3 = { + "page_content": None, + "content": "dict content fallback when page_content is None", + "id": "dict-3", + "score": 0.64, + } + + class CustomDuckDoc: + page_content = "duck content" + id = "duck-1" + score = 0.95 + + class CustomDuckDocContentFallback: + content = "duck content fallback" + id = "duck-2" + score = 0.81 + + class CustomDuckDocNonePageContentFallback: + page_content = None + content = "duck content fallback when page_content is None" + id = "duck-3" + score = 0.55 + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end( + documents=[ + dict_doc_1, + dict_doc_2, + dict_doc_3, + CustomDuckDoc(), + CustomDuckDocContentFallback(), + CustomDuckDocNonePageContentFallback(), + ], + run_id=run_id, + ) + + assigned = retrieval_inv.documents + assert len(assigned) == 6 + assert assigned[0] == { + "content": "dict content 1", + "id": "dict-1", + "score": 0.88, + } + assert assigned[1] == { + "content": "dict content 2", + "id": None, + "score": 0.72, + } + assert assigned[2] == { + "content": "dict content fallback when page_content is None", + "id": "dict-3", + "score": 0.64, + } + assert assigned[3] == { + "content": "duck content", + "id": "duck-1", + "score": 0.95, + } + assert assigned[4] == { + "content": "duck content fallback", + "id": "duck-2", + "score": 0.81, + } + assert assigned[5] == { + "content": "duck content fallback when page_content is None", + "id": "duck-3", + "score": 0.55, + } + + def test_document_score_non_mapping_metadata(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + class DuckDocNoMeta: + page_content = "text" + id = "doc-no-meta" + + class DuckDocStringMeta: + page_content = "text" + id = "doc-str-meta" + metadata = "not-a-mapping" + + class DuckDocNoneMeta: + page_content = "text" + id = "doc-none-meta" + metadata = None + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end( + documents=[ + DuckDocNoMeta(), + DuckDocStringMeta(), + DuckDocNoneMeta(), + ], + run_id=run_id, + ) + + for item in retrieval_inv.documents: + assert "score" not in item + + +class TestExtractDocumentScore: + def test_attribute_score(self): + class Obj: + score = 0.88 + + assert _extract_document_score(Obj()) == 0.88 + + def test_metadata_score(self): + class Obj: + metadata = {"score": 0.75} + + assert _extract_document_score(Obj()) == 0.75 + + def test_precedence_attr_over_metadata(self): + class Obj: + score = 0.9 + metadata = {"score": 0.4} + + assert _extract_document_score(Obj()) == 0.9 + + def test_attr_none_falls_back_to_metadata(self): + class Obj: + score = None + metadata = {"score": 0.65} + + assert _extract_document_score(Obj()) == 0.65 + + def test_zero_scores(self): + class ObjFloat: + score = 0.0 + + class ObjInt: + score = 0 + + assert _extract_document_score(ObjFloat()) == 0.0 + assert _extract_document_score(ObjInt()) == 0 + + def test_negative_score(self): + class Obj: + score = -1.25 + + assert _extract_document_score(Obj()) == -1.25 + + def test_non_numeric_and_bool(self): + class ObjBool: + score = True + + class ObjStr: + score = "0.9" + + assert _extract_document_score(ObjBool()) is None + assert _extract_document_score(ObjStr()) is None + + def test_non_finite_float_score(self): + class ObjNaN: + score = float("nan") + + class ObjInf: + score = float("inf") + + class ObjNegInf: + score = float("-inf") + + class ObjMathNaN: + score = math.nan + + class ObjMathInf: + score = math.inf + + class ObjMathNegInf: + score = -math.inf + + assert _extract_document_score(ObjNaN()) is None + assert _extract_document_score(ObjInf()) is None + assert _extract_document_score(ObjNegInf()) is None + assert _extract_document_score(ObjMathNaN()) is None + assert _extract_document_score(ObjMathInf()) is None + assert _extract_document_score(ObjMathNegInf()) is None + assert _extract_document_score({"score": float("nan")}) is None + assert _extract_document_score({"score": math.nan}) is None + assert ( + _extract_document_score({"metadata": {"score": float("inf")}}) + is None + ) + assert ( + _extract_document_score({"metadata": {"score": math.inf}}) is None + ) + assert ( + _extract_document_score({"metadata": {"score": float("-inf")}}) + is None + ) + assert ( + _extract_document_score({"metadata": {"score": -math.inf}}) is None + ) + + def test_dict_score(self): + assert _extract_document_score({"score": 0.82}) == 0.82 + assert _extract_document_score({"metadata": {"score": 0.91}}) == 0.91 + assert ( + _extract_document_score( + {"score": 0.99, "metadata": {"score": 0.1}} + ) + == 0.99 + ) + + def test_no_score(self): + assert _extract_document_score(object()) is None + assert _extract_document_score({}) is None + assert _extract_document_score({"metadata": None}) is None + assert _extract_document_score({"metadata": "str"}) is None + + +class TestDocumentToDict: + def test_document_with_page_content_and_score(self): + doc = Document( + page_content="doc content", id="d1", metadata={"score": 0.85} + ) + assert _document_to_dict(doc) == { + "content": "doc content", + "id": "d1", + "score": 0.85, + } + + def test_duck_typed_with_content_fallback_missing_page_content(self): + class DuckNoPageContent: + content = "fallback content" + id = "d2" + score = 0.9 + + assert _document_to_dict(DuckNoPageContent()) == { + "content": "fallback content", + "id": "d2", + "score": 0.9, + } + + def test_duck_typed_with_content_fallback_none_page_content(self): + class DuckNonePageContent: + page_content = None + content = "fallback content when page_content is None" + id = "d3" + score = 0.75 + + assert _document_to_dict(DuckNonePageContent()) == { + "content": "fallback content when page_content is None", + "id": "d3", + "score": 0.75, + } + + def test_mapping_with_content_fallback_missing_page_content(self): + doc_map = {"content": "mapping fallback", "id": "m1", "score": 0.8} + assert _document_to_dict(doc_map) == { + "content": "mapping fallback", + "id": "m1", + "score": 0.8, + } + + def test_mapping_with_content_fallback_none_page_content(self): + doc_map = { + "page_content": None, + "content": "mapping fallback when page_content is None", + "id": "m2", + "score": 0.7, + } + assert _document_to_dict(doc_map) == { + "content": "mapping fallback when page_content is None", + "id": "m2", + "score": 0.7, + } + + def test_non_finite_scores_omitted(self): + doc_nan = Document(page_content="c", metadata={"score": math.nan}) + doc_inf = Document(page_content="c", metadata={"score": float("inf")}) + doc_neginf = Document( + page_content="c", metadata={"score": float("-inf")} + ) + + assert _document_to_dict(doc_nan) == {"content": "c", "id": None} + assert _document_to_dict(doc_inf) == {"content": "c", "id": None} + assert _document_to_dict(doc_neginf) == {"content": "c", "id": None} + class TestOnRetrieverError: def test_invocation_failed(self): diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py index e79170afc..ef49b9618 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py @@ -11,6 +11,8 @@ from __future__ import annotations +import json +import math from typing import Any import pytest @@ -29,6 +31,12 @@ # --------------------------------------------------------------------------- +class _ScoredDocument(Document): + """Document subclass that allows a score attribute.""" + + score: float | None = None + + class _FakeRetriever(BaseRetriever): """In-memory retriever — no network calls, no embeddings.""" @@ -371,23 +379,25 @@ def test_document_metadata_not_in_span_content( meter_provider=meter_provider, logger_provider=logger_provider, ) - docs = [ - Document( - page_content="text", - metadata={"source": "wiki", "score": 0.9}, - ) - ] - retriever = _FakeRetriever(documents=docs) - retriever.invoke("q") + try: + docs = [ + Document( + page_content="text", + metadata={"source": "wiki", "author": "alice"}, + ) + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") - spans = span_exporter.get_finished_spans() - docs_attr = spans[0].attributes[ - gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS - ] - assert "text" in docs_attr - assert "wiki" not in docs_attr - assert "0.9" not in docs_attr - instrumentor.uninstrument() + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + assert "text" in docs_attr + assert "wiki" not in docs_attr + assert "alice" not in docs_attr + finally: + instrumentor.uninstrument() def test_empty_documents_in_span_content( @@ -409,13 +419,285 @@ def test_empty_documents_in_span_content( meter_provider=meter_provider, logger_provider=logger_provider, ) - retriever = _FakeRetriever(documents=[]) - retriever.invoke("q") + try: + retriever = _FakeRetriever(documents=[]) + retriever.invoke("q") - spans = span_exporter.get_finished_spans() - # documents attribute is set but represents an empty list - docs_attr = spans[0].attributes.get( - gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + spans = span_exporter.get_finished_spans() + # documents attribute is set but represents an empty list + docs_attr = spans[0].attributes.get( + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ) + assert docs_attr == "[]" + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_with_attribute_score( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" ) - assert docs_attr == "[]" - instrumentor.uninstrument() + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [_ScoredDocument(page_content="text", id="doc-1", score=0.85)] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert parsed[0]["content"] == "text" + assert parsed[0]["id"] == "doc-1" + assert parsed[0]["score"] == 0.85 + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_with_metadata_score( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [ + Document(page_content="text", id="doc-2", metadata={"score": 0.92}) + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert parsed[0]["content"] == "text" + assert parsed[0]["id"] == "doc-2" + assert parsed[0]["score"] == 0.92 + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_with_precedence( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [ + _ScoredDocument( + page_content="text", + id="doc-3", + score=0.9, + metadata={"score": 0.5}, + ) + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert parsed[0]["score"] == 0.9 + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_with_zero_score( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [_ScoredDocument(page_content="text", id="doc-4", score=0.0)] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert "score" in parsed[0] + assert parsed[0]["score"] == 0.0 + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_without_score( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [Document(page_content="text", id="doc-5")] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert "score" not in parsed[0] + finally: + instrumentor.uninstrument() + + +def test_retriever_content_capture_gating( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "NO_CONTENT" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [Document(page_content="text", metadata={"score": 0.95})] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert ( + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + not in spans[0].attributes + ) + finally: + instrumentor.uninstrument() + + +def test_retriever_documents_with_non_finite_scores( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [ + _ScoredDocument(page_content="c1", id="d1", score=float("nan")), + _ScoredDocument(page_content="c2", id="d2", score=float("inf")), + _ScoredDocument(page_content="c3", id="d3", score=float("-inf")), + Document(page_content="c4", id="d4", metadata={"score": math.nan}), + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + assert isinstance(docs_attr, str) + # RFC 8259 JSON compliance: unquoted NaN and Infinity must be strictly absent + assert "NaN" not in docs_attr + assert "Infinity" not in docs_attr + + parsed = json.loads(docs_attr) + assert len(parsed) == 4 + for item in parsed: + assert "score" not in item + finally: + instrumentor.uninstrument() From 29e7c69a8d53e6fc3c5581a0640257a12fc5ac56 Mon Sep 17 00:00:00 2001 From: dlowzzxx Date: Thu, 10 Sep 2026 09:39:25 +0200 Subject: [PATCH 2/4] docs(langchain): add towncrier changelog fragment for PR #670 --- .../.changelog/670.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/670.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/670.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/670.added new file mode 100644 index 000000000..fe116478e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/670.added @@ -0,0 +1 @@ +Capture document relevance score on retrieval spans per OpenTelemetry GenAI semantic conventions. From 9de659eb809ef7333b659bba6b1d09ebb9312d8e Mon Sep 17 00:00:00 2001 From: dlowzzxx Date: Fri, 11 Sep 2026 08:25:35 +0200 Subject: [PATCH 3/4] fix(langchain): support relevance_score fallback for document score extraction --- .../genai/langchain/callback_handler.py | 12 ++++++++++++ .../tests/test_callback_handler.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 693bfa423..2ada88fe7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -91,22 +91,34 @@ def _extract_document_score(doc: Any) -> float | int | None: if isinstance(doc, Mapping): doc_map = cast(Mapping[str, Any], doc) score = doc_map.get("score") + if score is None: + score = doc_map.get("relevance_score") if score is None: metadata = doc_map.get("metadata") if isinstance(metadata, Mapping): meta_map = cast(Mapping[str, Any], metadata) score = meta_map.get("score") + if score is None: + score = meta_map.get("relevance_score") elif metadata is not None: score = getattr(metadata, "score", None) + if score is None: + score = getattr(metadata, "relevance_score", None) else: score = getattr(doc, "score", None) + if score is None: + score = getattr(doc, "relevance_score", None) if score is None: metadata = getattr(doc, "metadata", None) if isinstance(metadata, Mapping): meta_map = cast(Mapping[str, Any], metadata) score = meta_map.get("score") + if score is None: + score = meta_map.get("relevance_score") elif metadata is not None: score = getattr(metadata, "score", None) + if score is None: + score = getattr(metadata, "relevance_score", None) if ( score is not None diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 8930745e6..eb7d762dd 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -1800,6 +1800,24 @@ class Obj: assert _extract_document_score(Obj()) == 0.75 + def test_metadata_relevance_score_fallback(self): + class Obj: + metadata = {"relevance_score": 0.82} + + assert _extract_document_score(Obj()) == 0.82 + + def test_attr_relevance_score_fallback(self): + class Obj: + relevance_score = 0.91 + + assert _extract_document_score(Obj()) == 0.91 + + def test_precedence_score_over_relevance_score(self): + class Obj: + metadata = {"score": 0.85, "relevance_score": 0.42} + + assert _extract_document_score(Obj()) == 0.85 + def test_precedence_attr_over_metadata(self): class Obj: score = 0.9 From 9607f37ef4b04fa89cf1ede713e4d892ffc750c0 Mon Sep 17 00:00:00 2001 From: dlowzzxx Date: Sun, 13 Sep 2026 22:24:42 +0200 Subject: [PATCH 4/4] docs(langchain): ground retriever score extraction with sync/async tests and documentation --- .../README.rst | 23 ++ .../genai/langchain/callback_handler.py | 13 +- .../tests/test_retriever.py | 232 +++++++++++++++++- 3 files changed, 265 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/README.rst b/instrumentation/opentelemetry-instrumentation-genai-langchain/README.rst index 4e24a50f8..94ae4992c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/README.rst @@ -16,6 +16,8 @@ application: * **Agent spans** for agent invocations nested inside a workflow, including the agent name, id, description, and conversation/session id when available. * **Tool spans** for tool calls made during a run. +* **Retrieval spans** for retriever invocations, capturing the query and retrieved + documents (including id, content, and relevance scores when available). The spans nest to reflect the graph, so a single graph invocation produces a workflow span with the agent, tool, and model calls it triggered as children. @@ -96,6 +98,27 @@ calls nested underneath. } ) +Retrieval Spans and Document Scores +----------------------------------- + +When invoking LangChain retrievers (e.g., vectorstores, knowledge bases, or contextual compression retrievers), +retrieval spans are recorded with the query and retrieved document metadata. + +When message content capture is enabled (``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_ONLY`` +or ``SPAN_AND_EVENT``), the retrieved documents are serialized into the +``gen_ai.retrieval.documents`` span attribute as a JSON array of objects with ``id`` and ``content``. + +When available, relevance and similarity scores are captured in each document object under ``score``: + +* **Direct retrieval scores**: extracted from ``metadata["score"]`` (populated by retrievers such as + ``AmazonKnowledgeBasesRetriever``, ``TavilySearchAPIRetriever``, and vectorstore score-threshold searches). +* **Reranking scores**: extracted from ``metadata["relevance_score"]`` (populated when retrievers are wrapped + with rerankers via ``ContextualCompressionRetriever``, such as ``CohereRerank``). +* **Duck-typed / custom documents**: extracted from a top-level ``score`` attribute or mapping key. + +If a document has no score, or if the score is non-numeric or non-finite (``NaN``, ``Infinity``), +the ``score`` key is omitted to ensure RFC 8259 JSON compliance. + Configuration ------------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 2ada88fe7..0211c6c90 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -84,8 +84,17 @@ def _conversation_id(metadata: dict[str, Any] | None) -> str | None: def _extract_document_score(doc: Any) -> float | int | None: """Extract relevance score polymorphically from a Document or Mapping. - Checks doc.score first, then falls back to doc.metadata['score']. - Also defensively supports Mapping/dict documents and duck-typed objects. + Retrieval scores are grounded in standard LangChain retrievers: + - Direct knowledge base and vector retrievers (e.g. AmazonKnowledgeBasesRetriever, + TavilySearchAPIRetriever) attach confidence/similarity scores to + ``doc.metadata["score"]``. + - Contextual compression retrievers wrapping rerankers (e.g. CohereRerank via + ContextualCompressionRetriever) populate ``doc.metadata["relevance_score"]``. + - Custom or duck-typed documents may provide a top-level ``score`` (or + ``relevance_score``) attribute or key. + + Non-finite floats (NaN, +/-Inf) and boolean values are filtered out to ensure + valid RFC 8259 JSON serialization in gen_ai.retrieval.documents. """ score: Any = None if isinstance(doc, Mapping): diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py index ef49b9618..27375f3b7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py @@ -16,7 +16,10 @@ from typing import Any import pytest -from langchain_core.callbacks import CallbackManagerForRetrieverRun +from langchain_core.callbacks import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever from pydantic import Field @@ -47,6 +50,11 @@ def _get_relevant_documents( ) -> list[Document]: return self.documents + async def _aget_relevant_documents( + self, query: str, *, run_manager: AsyncCallbackManagerForRetrieverRun + ) -> list[Document]: + return self.documents + def _get_ls_params(self, **kwargs: Any) -> Any: params = super()._get_ls_params(**kwargs) params["ls_vector_store_provider"] = "FakeVectorStore" @@ -61,6 +69,11 @@ def _get_relevant_documents( ) -> list[Document]: raise RuntimeError("retrieval failed") + async def _aget_relevant_documents( + self, query: str, *, run_manager: AsyncCallbackManagerForRetrieverRun + ) -> list[Document]: + raise RuntimeError("retrieval failed") + def _get_ls_params(self, **kwargs: Any) -> Any: params = super()._get_ls_params(**kwargs) params["ls_vector_store_provider"] = "FakeVectorStore" @@ -701,3 +714,220 @@ def test_retriever_documents_with_non_finite_scores( assert "score" not in item finally: instrumentor.uninstrument() + + +def test_retriever_documents_with_metadata_relevance_score( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [ + Document( + page_content="contextual content", + id="doc-rerank-1", + metadata={"relevance_score": 0.88}, + ) + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("query") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert parsed[0]["content"] == "contextual content" + assert parsed[0]["id"] == "doc-rerank-1" + assert parsed[0]["score"] == 0.88 + finally: + instrumentor.uninstrument() + + +@pytest.mark.asyncio +async def test_async_retriever_documents_with_scores( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + docs = [ + Document( + page_content="kb content", + id="doc-kb", + metadata={"score": 0.95}, + ), + Document( + page_content="rerank content", + id="doc-cohere", + metadata={"relevance_score": 0.82}, + ), + Document( + page_content="unscored content", + id="doc-plain", + metadata={"source": "plain.txt"}, + ), + ] + retriever = _FakeRetriever(documents=docs) + await retriever.ainvoke("async query") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 3 + + assert parsed[0]["id"] == "doc-kb" + assert parsed[0]["score"] == 0.95 + + assert parsed[1]["id"] == "doc-cohere" + assert parsed[1]["score"] == 0.82 + + assert parsed[2]["id"] == "doc-plain" + assert "score" not in parsed[2] + finally: + instrumentor.uninstrument() + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.asyncio +async def test_retriever_grounded_knowledge_base_sync_and_async( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, + is_async, +): + """Grounding test: models Bedrock Knowledge Bases / Tavily returning metadata['score'].""" + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + # Matches AmazonKnowledgeBasesRetriever / TavilySearchAPIRetriever document structure + kb_docs = [ + Document( + page_content="Amazon Bedrock Knowledge Bases provides managed RAG.", + id="kb-result-1", + metadata={ + "source": "s3://my-bucket/rag-guide.pdf", + "score": 0.89, + }, + ) + ] + retriever = _FakeRetriever(documents=kb_docs) + if is_async: + await retriever.ainvoke("what is bedrock rag?") + else: + retriever.invoke("what is bedrock rag?") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert ( + parsed[0]["content"] + == "Amazon Bedrock Knowledge Bases provides managed RAG." + ) + assert parsed[0]["id"] == "kb-result-1" + assert parsed[0]["score"] == 0.89 + finally: + instrumentor.uninstrument() + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.asyncio +async def test_retriever_grounded_contextual_compression_sync_and_async( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + monkeypatch, + is_async, +): + """Grounding test: models ContextualCompressionRetriever with CohereRerank returning metadata['relevance_score'].""" + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + # Matches CohereRerank.compress_documents inside ContextualCompressionRetriever + reranked_docs = [ + Document( + page_content="High relevance chunk after reranking.", + id="rerank-1", + metadata={ + "relevance_score": 0.94, + "model": "rerank-v3.5", + }, + ) + ] + retriever = _FakeRetriever(documents=reranked_docs) + if is_async: + await retriever.ainvoke("rerank query") + else: + retriever.invoke("rerank query") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + parsed = json.loads(docs_attr) + assert len(parsed) == 1 + assert parsed[0]["content"] == "High relevance chunk after reranking." + assert parsed[0]["id"] == "rerank-1" + assert parsed[0]["score"] == 0.94 + finally: + instrumentor.uninstrument()