From bb98f0b66d4f314addb281e77b03f63716a6aafc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:45:03 +0900 Subject: [PATCH 1/5] fix(telemetry): retain actionable probe failures Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 4 +++ contextual_orchestrator/provider_errors.py | 3 +- contextual_orchestrator/telemetry.py | 21 ++++++++++-- tests/test_provider_error_taxonomy.py | 16 +++++++++ tests/test_telemetry.py | 39 +++++++++++++++++++++- 5 files changed, 79 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 9221d43e3..39fdd6d5a 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -2018,6 +2018,10 @@ def _proxy_send( "gen_ai.provider.name": agent.provider_name or parsed_provider.hostname or agent.id, "gen_ai.request.model": agent.model, "contextual_orchestrator.agent_id": agent.id, + "contextual_orchestrator.model_group": agent.group_name or agent.model, + "contextual_orchestrator.fallback_outcome": ( + "not_attempted" if operation_kind == "capability_probe" else "not_observed" + ), "server.address": parsed_provider.hostname or "", "server.port": parsed_provider.port or (443 if parsed_provider.scheme == "https" else 80), } diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index ea47cee99..f159d1012 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -49,7 +49,8 @@ r"(?ix)(?:" r"https?://|" r"(?:^|[^0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:[^0-9]|$)|" - r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input|messages?)\b" + r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input)\b|" + r"\b(?:messages?|content)\s*(?:[:=]|\[|\{)" r")" ) diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index 3b9fdccb6..ce2f4c9d0 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -51,7 +51,11 @@ "gen_ai.usage.total_tokens", "contextual_orchestrator.agent_id", "contextual_orchestrator.error_code", + "contextual_orchestrator.error_summary", + "contextual_orchestrator.fallback_outcome", "contextual_orchestrator.latency_ms", + "contextual_orchestrator.model_group", + "contextual_orchestrator.operation_kind", "contextual_orchestrator.provider_status_code", "contextual_orchestrator.session_id_hash", "server.address", @@ -306,21 +310,34 @@ def traced( try: yield span except Exception as exc: - from .provider_errors import classify_provider_failure + from .provider_errors import classify_provider_failure, safe_provider_message classified = classify_provider_failure(exc, agent_id="", model="") failure_code = classified.error_code provider_status = classified.provider_status + error_summary = safe_provider_message(classified) or failure_code + model_group = safe.get("contextual_orchestrator.model_group", "ungrouped") + fallback_outcome = safe.get( + "contextual_orchestrator.fallback_outcome", "not_observed" + ) if Status is not None and StatusCode is not None: span.set_attribute("error.type", failure_code) + span.set_attribute( + "contextual_orchestrator.error_summary", error_summary + ) if provider_status is not None: span.set_attribute( "contextual_orchestrator.provider_status_code", provider_status ) span.set_status(Status(StatusCode.ERROR)) _LOGGER.warning( - "telemetry.operation_failed operation=%s error_type=%s", + "telemetry.operation_failed operation=%s error_type=%s " + "provider_status=%s error_summary=%r model_group=%s fallback_outcome=%s", name, failure_code, + provider_status, + error_summary, + model_group, + fallback_outcome, ) raise diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index af4a32ad8..b8ec08194 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -77,6 +77,22 @@ def test_safe_message_prefers_nested_provider_error_fields() -> None: assert detail == "validation failed" +def test_safe_message_keeps_actionable_schema_diagnostics_without_payloads() -> None: + """Schema field names are useful; field values and request bodies remain private.""" + actionable = "'messages' must contain the word 'json' to use json_object" + assert safe_provider_message( + _body_http_error(400, {"error": {"message": actionable}}) + ) == actionable + for diagnostic in ( + "messages=[{'role':'user','content':'customer secret'}]", + "prompt=customer secret", + "input: customer secret", + ): + assert safe_provider_message( + _body_http_error(400, {"error": {"message": diagnostic}}) + ) is None + + def test_safe_message_hides_unparseable_bodies_and_urls() -> None: """Non-JSON bodies return None so URLs/reasons never leak through fallback text.""" assert safe_provider_message(_http_error(500, b"upstream-secret http://10.0.0.9/internal")) is None diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 4d68f9cc7..f41016b2b 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -438,6 +438,7 @@ def capture(name, attributes): base_url="https://provider.example/v1", credential_key="", provider_name="openai", + group_name="model-family-x", ) monkeypatch.setattr(orchestrator_module, "traced", capture) monkeypatch.setattr(client, "_validate_provider", lambda unused_agent: None) @@ -448,6 +449,8 @@ def capture(name, attributes): assert client.probe_structured_chat(agent, {"messages": []}) == {"ok": True} assert captured[0][0] == "capability_probe.chat model-x" assert captured[0][1]["contextual_orchestrator.operation_kind"] == "capability_probe" + assert captured[0][1]["contextual_orchestrator.model_group"] == "model_family_x" + assert captured[0][1]["contextual_orchestrator.fallback_outcome"] == "not_attempted" def test_traced_starts_safe_client_span_with_error_type_and_no_raw_exception(monkeypatch, caplog): @@ -481,7 +484,11 @@ def test_traced_starts_safe_client_span_with_error_type_and_no_raw_exception(mon span.record_exception.assert_not_called() # Failures record the CLASSIFIED cause family (network timeout here), not # the Python exception class, and never the exception text. - span.set_attribute.assert_called_once_with("error.type", "provider_connection_error") + span.set_attribute.assert_any_call("error.type", "provider_connection_error") + span.set_attribute.assert_any_call( + "contextual_orchestrator.error_summary", + "the provider connection failed or did not finish in time", + ) assert "provider-response-secret" not in caplog.text assert "session-secret" not in caplog.text @@ -506,6 +513,36 @@ def test_traced_records_upstream_status_for_http_failures(monkeypatch): ) +def test_traced_logs_actionable_bounded_failure_evidence(monkeypatch, caplog): + """Operators get a safe cause, model group, status, and fallback outcome.""" + import urllib.error + + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer) + message = "'messages' must contain the word 'json' to use json_object" + body = io.BytesIO(json.dumps({"error": {"message": message}}).encode()) + + with pytest.raises(urllib.error.HTTPError): + with traced( + "capability_probe.chat gpt-4.1", + { + "contextual_orchestrator.model_group": "gpt-4.1", + "contextual_orchestrator.fallback_outcome": "not_attempted", + }, + ): + raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body) + + span.set_attribute.assert_any_call("error.type", "invalid_request_error") + span.set_attribute.assert_any_call("contextual_orchestrator.provider_status_code", 400) + span.set_attribute.assert_any_call("contextual_orchestrator.error_summary", message) + assert "provider_status=400" in caplog.text + assert "model_group=gpt-4.1" in caplog.text + assert "fallback_outcome=not_attempted" in caplog.text + assert message in caplog.text + assert "private.example" not in caplog.text + + def test_annotate_and_usage_helpers_filter_to_allowed_genai_attributes(): """Span annotation keeps approved scalars only; prompts never enter spans.""" span = MagicMock() From 50ea768e38d42eeacb8ac66d3b6842a8ddaacb69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:48:12 +0900 Subject: [PATCH 2/5] Restrict provider telemetry summaries --- contextual_orchestrator/telemetry.py | 26 +++++++++++++++++++++++++- tests/test_telemetry.py | 21 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index ce2f4c9d0..32b58731e 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -4,6 +4,8 @@ import hashlib import logging +import re +import urllib.error from collections.abc import Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar, Token @@ -39,6 +41,12 @@ # Match the OpenTelemetry SDK's default span-attribute budget so a single # sequence-valued attribute cannot exceed the span's default evidence budget. _MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128 +_SAFE_SCHEMA_DIAGNOSTIC = re.compile( + r"^['\"]?messages['\"]? must contain the word ['\"]?json['\"]?" + r"(?: in some form,)? to use " + r"(?:['\"]?response_format['\"]? of type ['\"]?json_object['\"]?|json_object)\.?$", + re.IGNORECASE, +) _ALLOWED_ATTRIBUTE_KEYS = frozenset( { "gen_ai.operation.name", @@ -315,7 +323,23 @@ def traced( classified = classify_provider_failure(exc, agent_id="", model="") failure_code = classified.error_code provider_status = classified.provider_status - error_summary = safe_provider_message(classified) or failure_code + # Arbitrary provider prose can echo caller content even when it has + # no assignment-shaped marker. Only one fixed-vocabulary schema + # diagnostic is safe enough to export; every other HTTP/provider + # failure uses the package-owned stable code. + provider_summary = ( + safe_provider_message(exc) + if isinstance(exc, urllib.error.HTTPError) + else None + ) + error_summary = ( + provider_summary + if provider_summary is not None + and _SAFE_SCHEMA_DIAGNOSTIC.fullmatch(provider_summary) + else failure_code + if isinstance(exc, urllib.error.HTTPError) + else safe_provider_message(classified) or failure_code + ) model_group = safe.get("contextual_orchestrator.model_group", "ungrouped") fallback_outcome = safe.get( "contextual_orchestrator.fallback_outcome", "not_observed" diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f41016b2b..ec9a72dc7 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -543,6 +543,27 @@ def test_traced_logs_actionable_bounded_failure_evidence(monkeypatch, caplog): assert "private.example" not in caplog.text +def test_traced_does_not_export_natural_language_provider_echo(monkeypatch, caplog): + """Unstructured provider prose is never evidence that request text is absent.""" + import urllib.error + + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer) + echoed = "The supplied phrase customer-private-text is not valid JSON" + body = io.BytesIO(json.dumps({"error": {"message": echoed}}).encode()) + + with pytest.raises(urllib.error.HTTPError): + with traced("capability_probe.chat model-x"): + raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body) + + span.set_attribute.assert_any_call( + "contextual_orchestrator.error_summary", "invalid_request_error" + ) + assert echoed not in caplog.text + assert "customer-private-text" not in caplog.text + + def test_annotate_and_usage_helpers_filter_to_allowed_genai_attributes(): """Span annotation keeps approved scalars only; prompts never enter spans.""" span = MagicMock() From 4e548b65b0018c43a3dd19ba79e513f1d301437a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:51:47 +0900 Subject: [PATCH 3/5] fix(security): reject quoted request fields in provider errors --- contextual_orchestrator/provider_errors.py | 2 +- tests/test_provider_error_taxonomy.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index f159d1012..f142e2430 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -50,7 +50,7 @@ r"https?://|" r"(?:^|[^0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:[^0-9]|$)|" r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input)\b|" - r"\b(?:messages?|content)\s*(?:[:=]|\[|\{)" + r"\b(?:messages?|content)[\"']?\s*(?:[:=]|\[|\{)" r")" ) diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index b8ec08194..1a3528923 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -85,6 +85,8 @@ def test_safe_message_keeps_actionable_schema_diagnostics_without_payloads() -> ) == actionable for diagnostic in ( "messages=[{'role':'user','content':'customer secret'}]", + '"messages": [{"role":"user","content":"customer secret"}]', + "'content': 'customer secret'", "prompt=customer secret", "input: customer secret", ): From ccbbaf132322433b7ebf3977739a49f99614cd0a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:53:09 +0900 Subject: [PATCH 4/5] fix(telemetry): close provider summary leaks Signed-off-by: Seongho Bae --- contextual_orchestrator/provider_errors.py | 2 +- contextual_orchestrator/telemetry.py | 25 ++++++------ tests/test_telemetry.py | 45 ++++++++++++++++++++-- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index f142e2430..40c95fbdd 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -50,7 +50,7 @@ r"https?://|" r"(?:^|[^0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:[^0-9]|$)|" r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input)\b|" - r"\b(?:messages?|content)[\"']?\s*(?:[:=]|\[|\{)" + r"(? Date: Tue, 1 Sep 2026 12:18:40 +0900 Subject: [PATCH 5/5] fix(telemetry): recognize prefixed schema failures Signed-off-by: Seongho Bae --- contextual_orchestrator/telemetry.py | 12 +++++++-- tests/test_telemetry.py | 37 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index eac87cdd1..45a4a2b2e 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import ipaddress import logging import re from collections.abc import Iterator, Mapping @@ -41,7 +42,7 @@ # sequence-valued attribute cannot exceed the span's default evidence budget. _MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128 _SAFE_SCHEMA_DIAGNOSTIC = re.compile( - r"^['\"]?messages['\"]? must contain the word ['\"]?json['\"]?" + r"['\"]?messages['\"]? must contain the word ['\"]?json['\"]?" r"(?: in some form,)? to use " r"(?:['\"]?response_format['\"]? of type ['\"]?json_object['\"]?|json_object)" r"(?:\.|$)", @@ -194,6 +195,13 @@ def _safe_attributes( if isinstance(value, (list, tuple)): continue if isinstance(value, str): + if key == "server.address": + try: + ipaddress.ip_address(value) + except ValueError: + pass + else: + continue result[key] = value[:256] elif isinstance(value, (bool, int, float)): result[key] = value @@ -334,7 +342,7 @@ def traced( error_summary = ( _SAFE_SCHEMA_ERROR_SUMMARY if provider_summary is not None - and _SAFE_SCHEMA_DIAGNOSTIC.match(provider_summary) + and _SAFE_SCHEMA_DIAGNOSTIC.search(provider_summary) else failure_code ) model_group = safe.get("contextual_orchestrator.model_group", "ungrouped") diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index adc351514..24c8740b0 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -59,6 +59,8 @@ def test_session_and_attribute_boundaries_reject_unsafe_values(): assert telemetry_module._normalize_session_id(value) is None assert session_id_from_metadata(None) is None assert telemetry_module._safe_attributes({"server.port": object()}) == {} + assert telemetry_module._safe_attributes({"server.address": "10.0.0.9"}) == {} + assert telemetry_module._safe_attributes({"server.address": "fd00::9"}) == {} token = set_session_id("session-safe") try: @@ -552,6 +554,41 @@ def test_traced_logs_actionable_bounded_failure_evidence(monkeypatch, caplog): assert "private.example" not in caplog.text +def test_traced_recognizes_litellm_prefixed_json_object_diagnostic( + monkeypatch, caplog +): + """A gateway prefix cannot hide Azure's actionable JSON-object contract.""" + import urllib.error + + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer) + message = ( + "AzureException BadRequestError - 'messages' must contain the word 'json' " + "in some form, to use 'response_format' of type 'json_object'." + "No fallback model group found; customer-private-text" + ) + body = io.BytesIO(json.dumps({"error": {"message": message}}).encode()) + + with pytest.raises(urllib.error.HTTPError): + with traced( + "capability_probe.chat gpt-4.1", + {"contextual_orchestrator.model_group": "gpt-4.1"}, + ): + raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body) + + span.set_attribute.assert_any_call( + "contextual_orchestrator.error_summary", + "messages must mention json when response_format is json_object", + ) + assert "provider_status=400" in caplog.text + assert "model_group=gpt-4.1" in caplog.text + assert "AzureException" not in caplog.text + assert "No fallback model group" not in caplog.text + assert "customer-private-text" not in caplog.text + assert "private.example" not in caplog.text + + def test_traced_does_not_export_natural_language_provider_echo(monkeypatch, caplog): """Unstructured provider prose is never evidence that request text is absent.""" import urllib.error