From dd04bd91c34bf28e58d62c64a3bbdfe921e39cc8 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 27 Jan 2026 14:53:33 -0800 Subject: [PATCH 01/11] Implement initial OTLP traces + trace stats with the .NET weblog example. Notably, this creates a new scenario APM_TRACING_OTLP to enable the environment variables needed to configure the SDK to export traces as OTLP. --- .github/workflows/run-end-to-end.yml | 5 + tests/otel/test_tracing_otlp.py | 132 ++++++++++++++++++ utils/_context/_scenarios/__init__.py | 16 +++ utils/_context/_scenarios/endtoend.py | 17 ++- utils/interfaces/_open_telemetry.py | 49 ++++++- utils/proxy/_deserializer.py | 2 + .../scripts/ci_orchestrators/workflow_data.py | 1 + 7 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 tests/otel/test_tracing_otlp.py diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 3e6ef05f7b4..6c3785efd2f 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -235,6 +235,11 @@ jobs: env: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} + - name: Run APM_TRACING_OTLP scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_OTLP"') + run: ./run.sh APM_TRACING_OTLP + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} - name: Run APM_TRACING_EFFICIENT_PAYLOAD scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_EFFICIENT_PAYLOAD"') run: ./run.sh APM_TRACING_EFFICIENT_PAYLOAD diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py new file mode 100644 index 00000000000..9c4721729e0 --- /dev/null +++ b/tests/otel/test_tracing_otlp.py @@ -0,0 +1,132 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +import json +from utils import weblog, interfaces, scenarios, features, incomplete_test_app +from utils._logger import logger +from typing import Any, Iterator + + +# Assert that the histogram has only one recorded data point matching the overall duration +def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], explicit_bounds: list[float]): + for i in range(len(explicit_bounds)): + is_first_index = i == 0 + is_last_index = i == len(explicit_bounds) - 1 + + if is_first_index and is_last_index: + assert bucket_counts[i] == 1 + assert duration <= explicit_bounds[i] + break + + if int(bucket_counts[i]) == 1: + lower_bound = float('-inf') if is_first_index else explicit_bounds[i-1] + upper_bound = float('inf') if is_last_index else explicit_bounds[i] + + if is_last_index: + assert duration > lower_bound and duration < upper_bound + else: + assert duration > lower_bound and duration <= upper_bound + +def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: + for keyValue in attributes: + if keyValue["value"].get("string_value"): + yield keyValue["key"], keyValue["value"]["string_value"] + elif keyValue["value"].get("stringValue"): + yield keyValue["key"], keyValue["value"]["stringValue"] + elif keyValue["value"].get("bool_value"): + yield keyValue["key"], keyValue["value"]["bool_value"] + elif keyValue["value"].get("boolValue"): + yield keyValue["key"], keyValue["value"]["boolValue"] + elif keyValue["value"].get("int_value"): + yield keyValue["key"], keyValue["value"]["int_value"] + elif keyValue["value"].get("intValue"): + yield keyValue["key"], keyValue["value"]["intValue"] + elif keyValue["value"].get("double_value"): + yield keyValue["key"], keyValue["value"]["double_value"] + elif keyValue["value"].get("doubleValue"): + yield keyValue["key"], keyValue["value"]["doubleValue"] + elif keyValue["value"].get("array_value"): + yield keyValue["key"], keyValue["value"]["array_value"] + elif keyValue["value"].get("arrayValue"): + yield keyValue["key"], keyValue["value"]["arrayValue"] + elif keyValue["value"].get("kvlist_value"): + yield keyValue["key"], keyValue["value"]["kvlist_value"] + elif keyValue["value"].get("kvlistValue"): + yield keyValue["key"], keyValue["value"]["kvlistValue"] + elif keyValue["value"].get("bytes_value"): + yield keyValue["key"], keyValue["value"]["bytes_value"] + elif keyValue["value"].get("bytesValue"): + yield keyValue["key"], keyValue["value"]["bytesValue"] + else: + raise ValueError(f"Unknown attribute value: {keyValue["value"]}") + + +# @scenarios.apm_tracing_e2e_otel +@features.otel_api +@scenarios.apm_tracing_otlp +class Test_Otel_Tracing_OTLP: + def setup_tracing(self): + self.req = weblog.get("/") + + # Note: Both camelcase and snake_case are allowed by the ProtoJSON Format (https://protobuf.dev/programming-guides/json/) + def test_tracing(self): + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + # _logger.debug(data) + assert len(data) == 1 + resource_span, span = data[0] + + # Assert resource attributes (we only expect string values) + attributes = {keyValue["key"]: keyValue["value"].get("string_value") or keyValue["value"].get("stringValue") for keyValue in resource_span.get("resource").get("attributes")} + assert attributes.get("service.name") == "weblog" + assert attributes.get("service.version") == "1.0.0" + assert attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" + # assert attributes.get("telemetry.sdk.name") == "datadog" + assert "telemetry.sdk.language" in attributes + assert "telemetry.sdk.version" in attributes + assert "git.commit.sha" in attributes + assert "git.repository_url" in attributes + assert "runtime-id" in attributes + + # Assert spans + assert span.get("name") == "GET /" + assert span.get("kind") == "SPAN_KIND_SERVER" + assert span.get("start_time_unix_nano") or span.get("startTimeUnixNano") + assert span.get("end_time_unix_nano") or span.get("endTimeUnixNano") + assert span.get("attributes") is not None + + # Assert HTTP tags + # Convert attributes list to a dictionary, but for now only handle KeyValue objects with stringValue + # attributes = {keyValue["key"]: keyValue["value"]["string_value"] or keyValue["value"]["stringValue"] for keyValue in span.get("attributes")} + span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) + assert span_attributes.get("http.method") == "GET" + assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later + assert span_attributes.get("http.url") == "http://localhost:7777/" + + # Assert trace stats + for metric, data_point in interfaces.open_telemetry.get_trace_stats("GET /"): + # If we run another export, we'll have the same aggregation keys with zero counts. Skip them. + if not data_point.get("sum"): + continue + + assert metric.get("name") == "request.latencies" + assert metric.get("description") == "Summary of request latencies" + assert metric.get("unit") == "ns" + assert metric.get("histogram").get("aggregationTemporality") == "AGGREGATION_TEMPORALITY_DELTA" + + assert int(data_point.get("count")) == 1 + assert sum(map(int, data_point.get("bucketCounts"))) == int(data_point.get("count")) + assert len(data_point.get("bucketCounts")) == len(data_point.get("explicitBounds")) + assert data_point.get("explicitBounds") == sorted(data_point.get("explicitBounds")) + assert_single_histogram_data_point(data_point.get("sum"), list(map(int, data_point.get("bucketCounts"))), data_point.get("explicitBounds")) + + attributes = dict(get_keyvalue_generator(data_point.get("attributes"))) + assert "Name" in attributes + assert attributes.get("Resource") == "GET /" + assert attributes.get("Type") == "web" + assert attributes.get("StatusCode") == "200" + assert attributes.get("TopLevel") == "True" + assert attributes.get("Error") == "False" + assert len(attributes) == 6 + + diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index ca1e06a7e9b..bb537381006 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -568,6 +568,22 @@ class _Scenarios: require_api_key=True, doc="", ) + apm_tracing_otlp = EndToEndScenario( + "APM_TRACING_OTLP", + weblog_env={ + "DD_TRACE_OTEL_ENABLED": "true", + "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-otlp-path=agent", + }, + backend_interface_timeout=5, + require_api_key=True, + include_opentelemetry=True, + doc="", + ) apm_tracing_efficient_payload = EndToEndScenario( "APM_TRACING_EFFICIENT_PAYLOAD", diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 8a4e41a6692..45456e462a7 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -244,6 +244,7 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_opentelemetry: bool = False, require_api_key: bool = False, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: @@ -323,6 +324,7 @@ def __init__( self.agent_interface_timeout = agent_interface_timeout self.backend_interface_timeout = backend_interface_timeout self._library_interface_timeout = library_interface_timeout + self.include_opentelemetry = include_opentelemetry def configure(self, config: pytest.Config): if self._require_api_key and "DD_API_KEY" not in os.environ and not self.replay: @@ -330,6 +332,7 @@ def configure(self, config: pytest.Config): self.weblog_infra.configure(config) self._set_containers_dependancies() + self.weblog_container.environment["DD_API_KEY"] = os.environ.get("DD_API_KEY") super().configure(config) interfaces.agent.configure(self.host_log_folder, replay=self.replay) @@ -339,6 +342,9 @@ def configure(self, config: pytest.Config): interfaces.library_stdout.configure(self.host_log_folder, replay=self.replay) interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) + if self.include_opentelemetry: + interfaces.open_telemetry.configure(self.host_log_folder, replay=self.replay) + for container in self.buddies: container.interface.configure(self.host_log_folder, replay=self.replay) @@ -412,7 +418,7 @@ def _get_weblog_system_info(self): def _start_interfaces_watchdog(self): super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.open_telemetry] if self.include_opentelemetry else [] ) def _set_weblog_domain(self): @@ -472,6 +478,10 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend.load_data_from_logs() + if self.include_opentelemetry: + interfaces.open_telemetry.load_data_from_logs() + interfaces.open_telemetry.check_deserialization_errors() + else: self._wait_interface( interfaces.library, 0 if force_interface_timout_to_zero else self.library_interface_timeout @@ -496,6 +506,11 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend, 0 if force_interface_timout_to_zero else self.backend_interface_timeout ) + if self.include_opentelemetry: + self._wait_interface( + interfaces.open_telemetry, 0 if force_interface_timout_to_zero else self.backend_interface_timeout + ) + def _wait_interface(self, interface: ProxyBasedInterfaceValidator, timeout: int): logger.terminal.write_sep("-", f"Wait for {interface} ({timeout}s)") logger.terminal.flush() diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 54c9381631c..f4acfb19db6 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -35,6 +35,53 @@ def get_otel_trace_id(self, request: HttpResponse): for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("stringValue") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") if attr_key == "http.request.headers.user-agent" and rid in attr_val: yield span.get("traceId") + elif attr_key == "http.useragent" and rid in attr_val: + yield span.get("traceId") + + def get_otel_spans(self, request: HttpResponse): + paths = ["/api/v0.2/traces", "/v1/traces"] + rid = request.get_rid() + + if rid: + logger.debug(f"Try to find traces related to request {rid}") + + for data in self.get_data(path_filters=paths): + resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get("content").get("resourceSpans") + for resource_span in resource_spans: + scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") + for scope_span in scope_spans: + for span in scope_span.get("spans"): + for attribute in span.get("attributes", []): + attr_key = attribute.get("key") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + if attr_key == "http.request.headers.user-agent" and rid in attr_val: + yield resource_span, span + break # Skip to next span + elif attr_key == "http.useragent" and rid in attr_val: + yield resource_span, span + break # Skip to next span + + def get_trace_stats(self, resource: str): + paths = ["/api/v0.2/stats", "/v1/metrics"] + + for data in self.get_data(path_filters=paths): + resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get("content").get("resourceMetrics") + if not resource_metrics: + continue + + for resource_metric in resource_metrics: + scope_metrics = resource_metric.get("scope_metrics") or resource_metric.get("scopeMetrics") + for scope_metric in scope_metrics: + if scope_metric.get("scope").get("name") == "datadog.trace.metrics": + for metric in scope_metric.get("metrics"): + if metric.get("name") == "request.latencies": + data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get("dataPoints") + for data_point in data_points: + for attribute in data_point.get("attributes", []): + attr_key = attribute.get("key") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + if attr_key == "Resource" and attr_val == resource: + yield metric, data_point diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 40b44f0303e..087c0e61895 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -191,6 +191,8 @@ def json_load(): return MessageToDict(ExportMetricsServiceResponse.FromString(content)) if path == "/v1/logs": return MessageToDict(ExportLogsServiceResponse.FromString(content)) + if path == "/api/v0.2/stats": + return MessageToDict(ExportMetricsServiceRequest.FromString(content)) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 008b42f44b2..7011c8aeed3 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -588,6 +588,7 @@ def _is_supported(library: str, weblog: str, scenario: str, _ci_environment: str "endtoend": [ "AGENT_NOT_SUPPORTING_SPAN_EVENTS", "APM_TRACING_E2E_OTEL", + "APM_TRACING_OTLP", "APM_TRACING_E2E_SINGLE_SPAN", "APPSEC_API_SECURITY", "APPSEC_API_SECURITY_NO_RESPONSE_BODY", From ebbbb5adefe3faa8d2d48d6a87fda741fe86e8eb Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:17:42 -0800 Subject: [PATCH 02/11] Skip assertions for the Trace Metrics, as we don't have a spec for this yet --- tests/otel/test_tracing_otlp.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 9c4721729e0..4007f892089 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -102,31 +102,3 @@ def test_tracing(self): assert span_attributes.get("http.method") == "GET" assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later assert span_attributes.get("http.url") == "http://localhost:7777/" - - # Assert trace stats - for metric, data_point in interfaces.open_telemetry.get_trace_stats("GET /"): - # If we run another export, we'll have the same aggregation keys with zero counts. Skip them. - if not data_point.get("sum"): - continue - - assert metric.get("name") == "request.latencies" - assert metric.get("description") == "Summary of request latencies" - assert metric.get("unit") == "ns" - assert metric.get("histogram").get("aggregationTemporality") == "AGGREGATION_TEMPORALITY_DELTA" - - assert int(data_point.get("count")) == 1 - assert sum(map(int, data_point.get("bucketCounts"))) == int(data_point.get("count")) - assert len(data_point.get("bucketCounts")) == len(data_point.get("explicitBounds")) - assert data_point.get("explicitBounds") == sorted(data_point.get("explicitBounds")) - assert_single_histogram_data_point(data_point.get("sum"), list(map(int, data_point.get("bucketCounts"))), data_point.get("explicitBounds")) - - attributes = dict(get_keyvalue_generator(data_point.get("attributes"))) - assert "Name" in attributes - assert attributes.get("Resource") == "GET /" - assert attributes.get("Type") == "web" - assert attributes.get("StatusCode") == "200" - assert attributes.get("TopLevel") == "True" - assert attributes.get("Error") == "False" - assert len(attributes) == 6 - - From 7fee5a2fc26c9aaa7a0e1308aeaca37162a05460 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:25:35 -0800 Subject: [PATCH 03/11] Run formatter --- tests/otel/test_tracing_otlp.py | 94 +++++++++++++++------------ utils/_context/_scenarios/endtoend.py | 6 +- utils/interfaces/_open_telemetry.py | 39 +++++++---- 3 files changed, 81 insertions(+), 58 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 4007f892089..746ed4c201e 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -2,10 +2,9 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. -import json -from utils import weblog, interfaces, scenarios, features, incomplete_test_app -from utils._logger import logger -from typing import Any, Iterator +from utils import weblog, interfaces, scenarios, features +from typing import Any +from collections.abc import Iterator # Assert that the histogram has only one recorded data point matching the overall duration @@ -20,46 +19,49 @@ def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], break if int(bucket_counts[i]) == 1: - lower_bound = float('-inf') if is_first_index else explicit_bounds[i-1] - upper_bound = float('inf') if is_last_index else explicit_bounds[i] + lower_bound = float("-inf") if is_first_index else explicit_bounds[i - 1] + upper_bound = float("inf") if is_last_index else explicit_bounds[i] if is_last_index: - assert duration > lower_bound and duration < upper_bound + assert duration > lower_bound + assert duration < upper_bound else: - assert duration > lower_bound and duration <= upper_bound + assert duration > lower_bound + assert duration <= upper_bound + def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: - for keyValue in attributes: - if keyValue["value"].get("string_value"): - yield keyValue["key"], keyValue["value"]["string_value"] - elif keyValue["value"].get("stringValue"): - yield keyValue["key"], keyValue["value"]["stringValue"] - elif keyValue["value"].get("bool_value"): - yield keyValue["key"], keyValue["value"]["bool_value"] - elif keyValue["value"].get("boolValue"): - yield keyValue["key"], keyValue["value"]["boolValue"] - elif keyValue["value"].get("int_value"): - yield keyValue["key"], keyValue["value"]["int_value"] - elif keyValue["value"].get("intValue"): - yield keyValue["key"], keyValue["value"]["intValue"] - elif keyValue["value"].get("double_value"): - yield keyValue["key"], keyValue["value"]["double_value"] - elif keyValue["value"].get("doubleValue"): - yield keyValue["key"], keyValue["value"]["doubleValue"] - elif keyValue["value"].get("array_value"): - yield keyValue["key"], keyValue["value"]["array_value"] - elif keyValue["value"].get("arrayValue"): - yield keyValue["key"], keyValue["value"]["arrayValue"] - elif keyValue["value"].get("kvlist_value"): - yield keyValue["key"], keyValue["value"]["kvlist_value"] - elif keyValue["value"].get("kvlistValue"): - yield keyValue["key"], keyValue["value"]["kvlistValue"] - elif keyValue["value"].get("bytes_value"): - yield keyValue["key"], keyValue["value"]["bytes_value"] - elif keyValue["value"].get("bytesValue"): - yield keyValue["key"], keyValue["value"]["bytesValue"] + for key_value in attributes: + if key_value["value"].get("string_value"): + yield key_value["key"], key_value["value"]["string_value"] + elif key_value["value"].get("stringValue"): + yield key_value["key"], key_value["value"]["stringValue"] + elif key_value["value"].get("bool_value"): + yield key_value["key"], key_value["value"]["bool_value"] + elif key_value["value"].get("boolValue"): + yield key_value["key"], key_value["value"]["boolValue"] + elif key_value["value"].get("int_value"): + yield key_value["key"], key_value["value"]["int_value"] + elif key_value["value"].get("intValue"): + yield key_value["key"], key_value["value"]["intValue"] + elif key_value["value"].get("double_value"): + yield key_value["key"], key_value["value"]["double_value"] + elif key_value["value"].get("doubleValue"): + yield key_value["key"], key_value["value"]["doubleValue"] + elif key_value["value"].get("array_value"): + yield key_value["key"], key_value["value"]["array_value"] + elif key_value["value"].get("arrayValue"): + yield key_value["key"], key_value["value"]["arrayValue"] + elif key_value["value"].get("kvlist_value"): + yield key_value["key"], key_value["value"]["kvlist_value"] + elif key_value["value"].get("kvlistValue"): + yield key_value["key"], key_value["value"]["kvlistValue"] + elif key_value["value"].get("bytes_value"): + yield key_value["key"], key_value["value"]["bytes_value"] + elif key_value["value"].get("bytesValue"): + yield key_value["key"], key_value["value"]["bytesValue"] else: - raise ValueError(f"Unknown attribute value: {keyValue["value"]}") + raise ValueError(f"Unknown attribute value: {key_value['value']}") # @scenarios.apm_tracing_e2e_otel @@ -77,10 +79,16 @@ def test_tracing(self): resource_span, span = data[0] # Assert resource attributes (we only expect string values) - attributes = {keyValue["key"]: keyValue["value"].get("string_value") or keyValue["value"].get("stringValue") for keyValue in resource_span.get("resource").get("attributes")} + attributes = { + key_value["key"]: key_value["value"].get("string_value") or key_value["value"].get("stringValue") + for key_value in resource_span.get("resource").get("attributes") + } assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" - assert attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" + assert ( + attributes.get("deployment.environment.name") == "system-tests" + or attributes.get("deployment.environment") == "system-tests" + ) # assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes @@ -96,9 +104,9 @@ def test_tracing(self): assert span.get("attributes") is not None # Assert HTTP tags - # Convert attributes list to a dictionary, but for now only handle KeyValue objects with stringValue - # attributes = {keyValue["key"]: keyValue["value"]["string_value"] or keyValue["value"]["stringValue"] for keyValue in span.get("attributes")} + # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue + # attributes = {key_value["key"]: key_value["value"]["string_value"] or key_value["value"]["stringValue"] for key_value in span.get("attributes")} span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) assert span_attributes.get("http.method") == "GET" - assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later + assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later assert span_attributes.get("http.url") == "http://localhost:7777/" diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 45456e462a7..c5a8be07738 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -418,7 +418,11 @@ def _get_weblog_system_info(self): def _start_interfaces_watchdog(self): super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.open_telemetry] if self.include_opentelemetry else [] + [interfaces.library, interfaces.agent] + + [container.interface for container in self.buddies] + + [interfaces.open_telemetry] + if self.include_opentelemetry + else [] ) def _set_weblog_domain(self): diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index f4acfb19db6..4abd7e75446 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -35,10 +35,12 @@ def get_otel_trace_id(self, request: HttpResponse): for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") - if attr_key == "http.request.headers.user-agent" and rid in attr_val: - yield span.get("traceId") - elif attr_key == "http.useragent" and rid in attr_val: + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( + "stringValue" + ) + if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( + attr_key == "http.useragent" and rid in attr_val + ): yield span.get("traceId") def get_otel_spans(self, request: HttpResponse): @@ -49,26 +51,31 @@ def get_otel_spans(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get("content").get("resourceSpans") + resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get( + "content" + ).get("resourceSpans") for resource_span in resource_spans: scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") for scope_span in scope_spans: for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") - if attr_key == "http.request.headers.user-agent" and rid in attr_val: - yield resource_span, span - break # Skip to next span - elif attr_key == "http.useragent" and rid in attr_val: + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( + "stringValue" + ) + if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( + attr_key == "http.useragent" and rid in attr_val + ): yield resource_span, span - break # Skip to next span + break # Skip to next span def get_trace_stats(self, resource: str): paths = ["/api/v0.2/stats", "/v1/metrics"] for data in self.get_data(path_filters=paths): - resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get("content").get("resourceMetrics") + resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( + "content" + ).get("resourceMetrics") if not resource_metrics: continue @@ -78,10 +85,14 @@ def get_trace_stats(self, resource: str): if scope_metric.get("scope").get("name") == "datadog.trace.metrics": for metric in scope_metric.get("metrics"): if metric.get("name") == "request.latencies": - data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get("dataPoints") + data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get( + "dataPoints" + ) for data_point in data_points: for attribute in data_point.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + attr_val = attribute.get("value").get("string_value") or attribute.get( + "value" + ).get("stringValue") if attr_key == "Resource" and attr_val == resource: yield metric, data_point From 5f8b2a8f60c06c6e4cbe1758acaca1e0da21ba32 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:41:51 -0800 Subject: [PATCH 04/11] Remove "/api/v0.2/stats" subpath for the OpenTelemetry interface so we only use the "/v1/metrics" subpath --- utils/interfaces/_open_telemetry.py | 2 +- utils/proxy/_deserializer.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 4abd7e75446..20049211ff8 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -70,7 +70,7 @@ def get_otel_spans(self, request: HttpResponse): break # Skip to next span def get_trace_stats(self, resource: str): - paths = ["/api/v0.2/stats", "/v1/metrics"] + paths = ["/v1/metrics"] for data in self.get_data(path_filters=paths): resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 087c0e61895..40b44f0303e 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -191,8 +191,6 @@ def json_load(): return MessageToDict(ExportMetricsServiceResponse.FromString(content)) if path == "/v1/logs": return MessageToDict(ExportLogsServiceResponse.FromString(content)) - if path == "/api/v0.2/stats": - return MessageToDict(ExportMetricsServiceRequest.FromString(content)) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) From 221b0318deedd997f192e0ba092914df52dc9300 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:45:04 -0800 Subject: [PATCH 05/11] Add manifest entries for languages --- manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + manifests/rust.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 58e72b9f79d..d0926d52bd1 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -734,6 +734,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecClusterEnabled: v3.36.0 tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecDisabledByDefault: v3.36.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: v3.9.0 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v2.42.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: missing_feature (does not support hostname) tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v3.4.1 diff --git a/manifests/golang.yml b/manifests/golang.yml index bb1ca16a981..ba55e32b560 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -921,6 +921,7 @@ manifest: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) net-http: v1.70.1 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.50.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v1.72.0-dev tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v1.67.0 diff --git a/manifests/java.yml b/manifests/java.yml index e56b7917251..c15a4fdfd11 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3250,6 +3250,7 @@ manifest: spring-boot-openliberty: v1.58.2+06122213c8 # Modified by easy win activation script uds-spring-boot: v1.58.2+06122213c8 # Modified by easy win activation script spring-boot-payara: v1.58.2+06122213c8 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.12.0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_generation_enabled_by_default: - declaration: missing_feature (Implemented in 1.24.0) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index a43e75b517f..6552447bfc6 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1723,6 +1723,7 @@ manifest: fastify: *ref_5_26_0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Node.js extract endpoint doesn't seem to be working.) tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_inject: incomplete_test_app (Node.js inject endpoint doesn't seem to be working.) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: *ref_3_0_0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_generation_enabled_by_default: - declaration: missing_feature (Implemented in 4.19.0 & 3.40.0) diff --git a/manifests/php.yml b/manifests/php.yml index d5794bb0498..d7497574d8b 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -539,6 +539,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingClusterOverride: v1.9.0 tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingDisabledByDefault: v1.9.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: incomplete_test_app (endpoint not implemented) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v0.84.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v1.9.0 tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v1.5.0 diff --git a/manifests/python.yml b/manifests/python.yml index ddcdff86a70..4546dc08f00 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1403,6 +1403,7 @@ manifest: flask-poc: v2.19.0 uds-flask: v4.3.1 # Modified by easy win activation script uwsgi-poc: v4.3.1 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v2.6.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v3.3.0.dev tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v2.15.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 81ae7be2154..a003e51624e 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1136,6 +1136,7 @@ manifest: "*": incomplete_test_app (endpoint not implemented) rails72: v2.0.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Ruby extract seems to fail even though it should be supported) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.17.0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_propagation_tid_chars: # Modified by easy win activation script - declaration: missing_feature (not implemented) diff --git a/manifests/rust.yml b/manifests/rust.yml index 9908665b134..6145325c5e8 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -10,6 +10,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v0.0.1 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3multi_128_bit_generation_disabled: missing_feature (propagation style not supported) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3multi_128_bit_generation_enabled: missing_feature (propagation style not supported) From cacbab5d2d9fbf80183186b48468c9a021593e24 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 14:10:50 -0800 Subject: [PATCH 06/11] Remove DD_TRACE_OTEL_ENABLED=true from the APM_TRACING_OTLP weblog env, since users do not need this feature for OTLP export to work --- utils/_context/_scenarios/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index bb537381006..50ff4a5fcd4 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -571,7 +571,6 @@ class _Scenarios: apm_tracing_otlp = EndToEndScenario( "APM_TRACING_OTLP", weblog_env={ - "DD_TRACE_OTEL_ENABLED": "true", "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "OTEL_TRACES_EXPORTER": "otlp", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", From 14b1995be35068bf065d332c99bcdcb611fc6b9e Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 14:13:07 -0800 Subject: [PATCH 07/11] Remove all helper methods and setup related to OTLP trace stats, since we're not immediately implementing that --- tests/otel/test_tracing_otlp.py | 23 ---------------------- utils/_context/_scenarios/__init__.py | 3 --- utils/interfaces/_open_telemetry.py | 28 --------------------------- 3 files changed, 54 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 746ed4c201e..fac734d707f 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -7,29 +7,6 @@ from collections.abc import Iterator -# Assert that the histogram has only one recorded data point matching the overall duration -def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], explicit_bounds: list[float]): - for i in range(len(explicit_bounds)): - is_first_index = i == 0 - is_last_index = i == len(explicit_bounds) - 1 - - if is_first_index and is_last_index: - assert bucket_counts[i] == 1 - assert duration <= explicit_bounds[i] - break - - if int(bucket_counts[i]) == 1: - lower_bound = float("-inf") if is_first_index else explicit_bounds[i - 1] - upper_bound = float("inf") if is_last_index else explicit_bounds[i] - - if is_last_index: - assert duration > lower_bound - assert duration < upper_bound - else: - assert duration > lower_bound - assert duration <= upper_bound - - def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 50ff4a5fcd4..52da2baad53 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -571,12 +571,9 @@ class _Scenarios: apm_tracing_otlp = EndToEndScenario( "APM_TRACING_OTLP", weblog_env={ - "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "OTEL_TRACES_EXPORTER": "otlp", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics", - "OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-otlp-path=agent", }, backend_interface_timeout=5, require_api_key=True, diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 20049211ff8..03f7fca030a 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -68,31 +68,3 @@ def get_otel_spans(self, request: HttpResponse): ): yield resource_span, span break # Skip to next span - - def get_trace_stats(self, resource: str): - paths = ["/v1/metrics"] - - for data in self.get_data(path_filters=paths): - resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( - "content" - ).get("resourceMetrics") - if not resource_metrics: - continue - - for resource_metric in resource_metrics: - scope_metrics = resource_metric.get("scope_metrics") or resource_metric.get("scopeMetrics") - for scope_metric in scope_metrics: - if scope_metric.get("scope").get("name") == "datadog.trace.metrics": - for metric in scope_metric.get("metrics"): - if metric.get("name") == "request.latencies": - data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get( - "dataPoints" - ) - for data_point in data_points: - for attribute in data_point.get("attributes", []): - attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get( - "value" - ).get("stringValue") - if attr_key == "Resource" and attr_val == resource: - yield metric, data_point From 57ed6bc04c4f7f297d3d5916a8778ea8422bad3c Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 18:25:00 -0800 Subject: [PATCH 08/11] Add more comprehensive testing, especially around the http/json encoding specifically: - At runtime determine if the request is JSON - If JSON, look up proto field names by their camelCase representation. Otherwise, look up field names by their snake_case representation - If JSON, assert that the 'traceId' and 'spanId' fields are case-insensitive hexadecimal strings, rather than base64-encoded strings - If JSON, assert that enums (e.g. span.kind and span.status.code) are encoded using an integer, not a string representation of the enum value name - Regardless of protocol, get the time before and after the test HTTP request is issued, and assert that the span's reported 'start_time_unix_nano' and 'end_time_unix_nano' fall in this range - Regardless of protocol, make the 'http.method' and 'http.status_code' span attribute assertions more flexible by also testing against their stable OpenTelemetry HTTP equivalents of 'http.request.method' and 'http.response.status_code', respectively --- tests/otel/test_tracing_otlp.py | 112 +++++++++++++++++++++++----- utils/interfaces/_open_telemetry.py | 7 +- 2 files changed, 95 insertions(+), 24 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index fac734d707f..d833ce00e7e 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -2,11 +2,46 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. +import time +import re +from enum import Enum from utils import weblog, interfaces, scenarios, features from typing import Any from collections.abc import Iterator +def _snake_to_camel(snake_key: str) -> str: + parts = snake_key.split("_") + return parts[0].lower() + "".join(p.capitalize() for p in parts[1:]) + + +def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool, default: Any = None) -> Any: # noqa: ANN401 + """Look up a field by its snake_case name when is_json is false, or its camelCase equivalent when is_json is true. + Fields must be camelCase for JSON Protobuf encoding. See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + """ + if d is None: + return default + key = _snake_to_camel(snake_case_key) if is_json else snake_case_key + return d.get(key, default) + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 +class SpanKind(Enum): + UNSPECIFIED = 0 + INTERNAL = 1 + SERVER = 2 + CLIENT = 3 + PRODUCER = 4 + CONSUMER = 5 + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 +class StatusCode(Enum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 + + def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): @@ -45,45 +80,82 @@ def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: @features.otel_api @scenarios.apm_tracing_otlp class Test_Otel_Tracing_OTLP: - def setup_tracing(self): + def setup_single_server_trace(self): + self.start_time_ns = time.time_ns() self.req = weblog.get("/") + self.end_time_ns = time.time_ns() - # Note: Both camelcase and snake_case are allowed by the ProtoJSON Format (https://protobuf.dev/programming-guides/json/) - def test_tracing(self): + def test_single_server_trace(self): data = list(interfaces.open_telemetry.get_otel_spans(self.req)) - # _logger.debug(data) + + # Assert that there is only one OTLP request containing the desired server span assert len(data) == 1 - resource_span, span = data[0] + request, content, span = data[0] + + # Determine if JSON Protobuf Encoding was used for the OTLP request (rather than Binary Protobuf) + # We need to assert that we match the OTLP specification, which has some odd encoding rules when using JSON: https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + request_headers = {key.lower(): value for key, value in request.get("headers")} + is_json = request_headers.get("content-type") == "application/json" + + # Assert that there is only one resource span (i.e. SDK) in the OTLP request + resource_spans = get_otlp_key(content, "resource_spans", is_json=is_json) + expected_key = _snake_to_camel("resource_spans") if is_json else "resource_spans" + assert resource_spans is not None, f"missing '{expected_key}' on content: {content}" + assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" + resource_span = resource_spans[0] - # Assert resource attributes (we only expect string values) attributes = { - key_value["key"]: key_value["value"].get("string_value") or key_value["value"].get("stringValue") + key_value["key"]: get_otlp_key(key_value["value"], "string_value", is_json=is_json) for key_value in resource_span.get("resource").get("attributes") } + + # Assert that the resource attributes contain the service-level attributes that were configured for weblog assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" assert ( attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) + + # Assert that the resource attributes contain the tracer-level attributes we expect # assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes - assert "git.commit.sha" in attributes - assert "git.repository_url" in attributes assert "runtime-id" in attributes + # assert "git.commit.sha" in attributes + # assert "git.repository_url" in attributes - # Assert spans - assert span.get("name") == "GET /" - assert span.get("kind") == "SPAN_KIND_SERVER" - assert span.get("start_time_unix_nano") or span.get("startTimeUnixNano") - assert span.get("end_time_unix_nano") or span.get("endTimeUnixNano") - assert span.get("attributes") is not None + # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. + # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + if is_json: + assert re.match(r"^[0-9a-fA-F]{32}$", span.get("traceId")), ( + f"traceId is not a valid case-insensitive hexadecimal string, got {span.get('traceId')}" + ) + assert re.match(r"^[0-9a-fA-F]{16}$", span.get("spanId")), ( + f"spanId is not a valid case-insensitive hexadecimal string, got {span.get('spanId')}" + ) + + # Assert that the span fields match the expected values + span_start_time_ns = int(get_otlp_key(span, "start_time_unix_nano", is_json=is_json)) + span_end_time_ns = int(get_otlp_key(span, "end_time_unix_nano", is_json=is_json)) + assert span_start_time_ns >= self.start_time_ns + assert span_end_time_ns >= span_start_time_ns + assert span_end_time_ns <= self.end_time_ns + + assert get_otlp_key(span, "name", is_json=is_json) + assert get_otlp_key(span, "kind", is_json=is_json) == SpanKind.SERVER.value + assert get_otlp_key(span, "attributes", is_json=is_json) is not None + assert ( + get_otlp_key(span, "status", is_json=is_json) is None + or get_otlp_key(span, "status", is_json=is_json).get("code") == StatusCode.STATUS_CODE_UNSET.value + ) # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue - # attributes = {key_value["key"]: key_value["value"]["string_value"] or key_value["value"]["stringValue"] for key_value in span.get("attributes")} - span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) - assert span_attributes.get("http.method") == "GET" - assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later - assert span_attributes.get("http.url") == "http://localhost:7777/" + span_attributes = dict(get_keyvalue_generator(get_otlp_key(span, "attributes", is_json=is_json))) + method = span_attributes.get("http.method") or span_attributes.get("http.request.method") + status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") + assert method == "GET", f"HTTP method is not GET, got {method}" + assert status_code is not None + assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" + # assert span_attributes.get("http.url") == "http://localhost:7777/" diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 03f7fca030a..23836c50a99 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -51,9 +51,8 @@ def get_otel_spans(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get( - "content" - ).get("resourceSpans") + content = data.get("request").get("content") + resource_spans = content.get("resource_spans") or content.get("resourceSpans") for resource_span in resource_spans: scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") for scope_span in scope_spans: @@ -66,5 +65,5 @@ def get_otel_spans(self, request: HttpResponse): if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( attr_key == "http.useragent" and rid in attr_val ): - yield resource_span, span + yield data.get("request"), content, span break # Skip to next span From 6a1df0763c23d61ede7c70d6663145092c66b4b9 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 18:30:20 -0800 Subject: [PATCH 09/11] Small refactoring of code and comments --- tests/otel/test_tracing_otlp.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index d833ce00e7e..b881a9ee093 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -109,24 +109,21 @@ def test_single_server_trace(self): for key_value in resource_span.get("resource").get("attributes") } - # Assert that the resource attributes contain the service-level attributes that were configured for weblog + # Assert that the resource attributes contain the service-level attributes and tracer-level attributes we expect + # TODO: Assert the following attributes: runtime-id, git.commit.sha, git.repository_url assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" assert ( attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) - - # Assert that the resource attributes contain the tracer-level attributes we expect - # assert attributes.get("telemetry.sdk.name") == "datadog" + assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes - assert "runtime-id" in attributes - # assert "git.commit.sha" in attributes - # assert "git.repository_url" in attributes # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + # TODO: Assert against trace_id and span_id fields in the protobuf encoding as well if is_json: assert re.match(r"^[0-9a-fA-F]{32}$", span.get("traceId")), ( f"traceId is not a valid case-insensitive hexadecimal string, got {span.get('traceId')}" @@ -158,4 +155,3 @@ def test_single_server_trace(self): assert method == "GET", f"HTTP method is not GET, got {method}" assert status_code is not None assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" - # assert span_attributes.get("http.url") == "http://localhost:7777/" From 746148b7ed59967bc0e6f0a104ab52c03091990b Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 10 Mar 2026 14:39:09 -0700 Subject: [PATCH 10/11] Add test case asserting that an incoming trace context's sampling decision of 0 is respected for OTLP traces by default --- tests/otel/test_tracing_otlp.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index b881a9ee093..2ec2498bfa0 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -86,6 +86,7 @@ def setup_single_server_trace(self): self.end_time_ns = time.time_ns() def test_single_server_trace(self): + """Validates the required elements of the OTLP payload for a single trace""" data = list(interfaces.open_telemetry.get_otel_spans(self.req)) # Assert that there is only one OTLP request containing the desired server span @@ -155,3 +156,13 @@ def test_single_server_trace(self): assert method == "GET", f"HTTP method is not GET, got {method}" assert status_code is not None assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" + + def setup_unsampled_trace(self): + self.req = weblog.get("/", headers={"traceparent": "00-11111111111111110000000000000001-0000000000000001-00"}) + + def test_unsampled_trace(self): + """Validates that the spans from a non-sampled trace are not exported.""" + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + + # Assert that the span from this test case was not exported + assert len(data) == 0, f"Expected no weblog spans in the OTLP trace payload, got {data}" From 919b5efde5b7498e2477caab3065ae49b58ace07 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 10 Mar 2026 14:45:54 -0700 Subject: [PATCH 11/11] Move enums to dd_constants.py --- tests/otel/test_tracing_otlp.py | 19 +------------------ utils/dd_constants.py | 8 ++++++++ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 2ec2498bfa0..562949d3e37 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -4,8 +4,8 @@ import time import re -from enum import Enum from utils import weblog, interfaces, scenarios, features +from utils.dd_constants import SpanKind, StatusCode from typing import Any from collections.abc import Iterator @@ -25,23 +25,6 @@ def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool return d.get(key, default) -# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 -class SpanKind(Enum): - UNSPECIFIED = 0 - INTERNAL = 1 - SERVER = 2 - CLIENT = 3 - PRODUCER = 4 - CONSUMER = 5 - - -# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 -class StatusCode(Enum): - STATUS_CODE_UNSET = 0 - STATUS_CODE_OK = 1 - STATUS_CODE_ERROR = 2 - - def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): diff --git a/utils/dd_constants.py b/utils/dd_constants.py index b2bf0e3b539..507704bee78 100644 --- a/utils/dd_constants.py +++ b/utils/dd_constants.py @@ -108,6 +108,7 @@ class SamplingMechanism(IntEnum): AI_GUARD = 13 +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 class SpanKind(IntEnum): UNSPECIFIED = 0 INTERNAL = 1 @@ -115,3 +116,10 @@ class SpanKind(IntEnum): CLIENT = 3 PRODUCER = 4 CONSUMER = 5 + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 +class StatusCode(IntEnum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2