fix(telemetry): retain actionable provider failure evidence

contextual_orchestrator/orchestrator.py

@@ -2018,6 +2018,10 @@ def _proxy_send(
             "gen_ai.provider.name": agent.provider_name or parsed_provider.hostname or agent.id,
             "gen_ai.request.model": agent.model,
             "contextual_orchestrator.agent_id": agent.id,
+            "contextual_orchestrator.model_group": agent.group_name or agent.model,
+            "contextual_orchestrator.fallback_outcome": (
+                "not_attempted" if operation_kind == "capability_probe" else "not_observed"
+            ),
             "server.address": parsed_provider.hostname or "",
             "server.port": parsed_provider.port or (443 if parsed_provider.scheme == "https" else 80),
         }

contextual_orchestrator/provider_errors.py

@@ -49,7 +49,8 @@
     r"(?ix)(?:"
     r"https?://|"
     r"(?:^|[^0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:[^0-9]|$)|"
-    r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input|messages?)\b"
+    r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input)\b|"
+    r"(?<!\w)['\"]?(?:messages?|content)['\"]?\s*(?:[:=]|\[|\{)"
     r")"
 )
 

contextual_orchestrator/telemetry.py

@@ -3,7 +3,9 @@
 from __future__ import annotations
 
 import hashlib
+import ipaddress
 import logging
+import re
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar, Token
@@ -39,6 +41,16 @@
 # Match the OpenTelemetry SDK's default span-attribute budget so a single
 # sequence-valued attribute cannot exceed the span's default evidence budget.
 _MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128
+_SAFE_SCHEMA_DIAGNOSTIC = re.compile(
+    r"['\"]?messages['\"]? must contain the word ['\"]?json['\"]?"
+    r"(?: in some form,)? to use "
+    r"(?:['\"]?response_format['\"]? of type ['\"]?json_object['\"]?|json_object)"
+    r"(?:\.|$)",
+    re.IGNORECASE,
+)
+_SAFE_SCHEMA_ERROR_SUMMARY = (
+    "messages must mention json when response_format is json_object"
+)
 _ALLOWED_ATTRIBUTE_KEYS = frozenset(
     {
         "gen_ai.operation.name",
@@ -51,7 +63,11 @@
         "gen_ai.usage.total_tokens",
         "contextual_orchestrator.agent_id",
         "contextual_orchestrator.error_code",
+        "contextual_orchestrator.error_summary",
+        "contextual_orchestrator.fallback_outcome",
         "contextual_orchestrator.latency_ms",
+        "contextual_orchestrator.model_group",
+        "contextual_orchestrator.operation_kind",
         "contextual_orchestrator.provider_status_code",
         "contextual_orchestrator.session_id_hash",
         "server.address",
@@ -179,6 +195,13 @@ def _safe_attributes(
         if isinstance(value, (list, tuple)):
             continue
         if isinstance(value, str):
+            if key == "server.address":
+                try:
+                    ipaddress.ip_address(value)
+                except ValueError:
+                    pass
+                else:
+                    continue
             result[key] = value[:256]
         elif isinstance(value, (bool, int, float)):
             result[key] = value
@@ -306,21 +329,44 @@ def traced(
         try:
             yield span
         except Exception as exc:
-            from .provider_errors import classify_provider_failure
+            from .provider_errors import classify_provider_failure, safe_provider_message
 
             classified = classify_provider_failure(exc, agent_id="", model="")
             failure_code = classified.error_code
             provider_status = classified.provider_status
+            # Arbitrary provider prose can echo caller content even when it has
+            # no assignment-shaped marker. Recognize one exact schema contract,
+            # export our fixed wording, and reduce everything else to its stable
+            # package-owned code.
+            provider_summary = safe_provider_message(exc)
+            error_summary = (
+                _SAFE_SCHEMA_ERROR_SUMMARY
+                if provider_summary is not None
+                and _SAFE_SCHEMA_DIAGNOSTIC.search(provider_summary)
+                else failure_code
+            )
+            model_group = safe.get("contextual_orchestrator.model_group", "ungrouped")
+            fallback_outcome = safe.get(
+                "contextual_orchestrator.fallback_outcome", "not_observed"
+            )
             if Status is not None and StatusCode is not None:
                 span.set_attribute("error.type", failure_code)
+                span.set_attribute(
+                    "contextual_orchestrator.error_summary", error_summary
+                )
                 if provider_status is not None:
                     span.set_attribute(
                         "contextual_orchestrator.provider_status_code", provider_status
                     )
                 span.set_status(Status(StatusCode.ERROR))
             _LOGGER.warning(
-                "telemetry.operation_failed operation=%s error_type=%s",
+                "telemetry.operation_failed operation=%s error_type=%s "
+                "provider_status=%s error_summary=%r model_group=%s fallback_outcome=%s",
                 name,
                 failure_code,
+                provider_status,
+                error_summary,
+                model_group,
+                fallback_outcome,
             )
             raise

tests/test_provider_error_taxonomy.py

@@ -77,6 +77,24 @@ def test_safe_message_prefers_nested_provider_error_fields() -> None:
     assert detail == "validation failed"
 
 
+def test_safe_message_keeps_actionable_schema_diagnostics_without_payloads() -> None:
+    """Schema field names are useful; field values and request bodies remain private."""
+    actionable = "'messages' must contain the word 'json' to use json_object"
+    assert safe_provider_message(
+        _body_http_error(400, {"error": {"message": actionable}})
+    ) == actionable
+    for diagnostic in (
+        "messages=[{'role':'user','content':'customer secret'}]",
+        '"messages": [{"role":"user","content":"customer secret"}]',
+        "'content': 'customer secret'",
+        "prompt=customer secret",
+        "input: customer secret",
+    ):
+        assert safe_provider_message(
+            _body_http_error(400, {"error": {"message": diagnostic}})
+        ) is None
+
+
 def test_safe_message_hides_unparseable_bodies_and_urls() -> None:
     """Non-JSON bodies return None so URLs/reasons never leak through fallback text."""
     assert safe_provider_message(_http_error(500, b"upstream-secret http://10.0.0.9/internal")) is None

tests/test_telemetry.py

@@ -59,6 +59,8 @@ def test_session_and_attribute_boundaries_reject_unsafe_values():
         assert telemetry_module._normalize_session_id(value) is None
     assert session_id_from_metadata(None) is None
     assert telemetry_module._safe_attributes({"server.port": object()}) == {}
+    assert telemetry_module._safe_attributes({"server.address": "10.0.0.9"}) == {}
+    assert telemetry_module._safe_attributes({"server.address": "fd00::9"}) == {}
 
     token = set_session_id("session-safe")
     try:
@@ -438,6 +440,7 @@ def capture(name, attributes):
         base_url="https://provider.example/v1",
         credential_key="",
         provider_name="openai",
+        group_name="model-family-x",
     )
     monkeypatch.setattr(orchestrator_module, "traced", capture)
     monkeypatch.setattr(client, "_validate_provider", lambda unused_agent: None)
@@ -448,6 +451,8 @@ def capture(name, attributes):
     assert client.probe_structured_chat(agent, {"messages": []}) == {"ok": True}
     assert captured[0][0] == "capability_probe.chat model-x"
     assert captured[0][1]["contextual_orchestrator.operation_kind"] == "capability_probe"
+    assert captured[0][1]["contextual_orchestrator.model_group"] == "model_family_x"
+    assert captured[0][1]["contextual_orchestrator.fallback_outcome"] == "not_attempted"
 
 
 def test_traced_starts_safe_client_span_with_error_type_and_no_raw_exception(monkeypatch, caplog):
@@ -481,7 +486,11 @@ def test_traced_starts_safe_client_span_with_error_type_and_no_raw_exception(mon
     span.record_exception.assert_not_called()
     # Failures record the CLASSIFIED cause family (network timeout here), not
     # the Python exception class, and never the exception text.
-    span.set_attribute.assert_called_once_with("error.type", "provider_connection_error")
+    span.set_attribute.assert_any_call("error.type", "provider_connection_error")
+    span.set_attribute.assert_any_call(
+        "contextual_orchestrator.error_summary",
+        "provider_connection_error",
+    )
     assert "provider-response-secret" not in caplog.text
     assert "session-secret" not in caplog.text
 
@@ -506,6 +515,129 @@ def test_traced_records_upstream_status_for_http_failures(monkeypatch):
     )
 
 
+def test_traced_logs_actionable_bounded_failure_evidence(monkeypatch, caplog):
+    """Operators get a safe cause, model group, status, and fallback outcome."""
+    import urllib.error
+
+    tracer = MagicMock()
+    span = tracer.start_as_current_span.return_value.__enter__.return_value
+    monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer)
+    message = (
+        "'messages' must contain the word 'json' in some form, to use "
+        "'response_format' of type 'json_object'.No fallback model group found; "
+        "customer-private-text"
+    )
+    body = io.BytesIO(json.dumps({"error": {"message": message}}).encode())
+
+    with pytest.raises(urllib.error.HTTPError):
+        with traced(
+            "capability_probe.chat gpt-4.1",
+            {
+                "contextual_orchestrator.model_group": "gpt-4.1",
+                "contextual_orchestrator.fallback_outcome": "not_attempted",
+            },
+        ):
+            raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body)
+
+    span.set_attribute.assert_any_call("error.type", "invalid_request_error")
+    span.set_attribute.assert_any_call("contextual_orchestrator.provider_status_code", 400)
+    span.set_attribute.assert_any_call(
+        "contextual_orchestrator.error_summary",
+        "messages must mention json when response_format is json_object",
+    )
+    assert "provider_status=400" in caplog.text
+    assert "model_group=gpt-4.1" in caplog.text
+    assert "fallback_outcome=not_attempted" in caplog.text
+    assert "messages must mention json when response_format is json_object" in caplog.text
+    assert "No fallback model group" not in caplog.text
+    assert "customer-private-text" not in caplog.text
+    assert "private.example" not in caplog.text
+
+
+def test_traced_recognizes_litellm_prefixed_json_object_diagnostic(
+    monkeypatch, caplog
+):
+    """A gateway prefix cannot hide Azure's actionable JSON-object contract."""
+    import urllib.error
+
+    tracer = MagicMock()
+    span = tracer.start_as_current_span.return_value.__enter__.return_value
+    monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer)
+    message = (
+        "AzureException BadRequestError - 'messages' must contain the word 'json' "
+        "in some form, to use 'response_format' of type 'json_object'."
+        "No fallback model group found; customer-private-text"
+    )
+    body = io.BytesIO(json.dumps({"error": {"message": message}}).encode())
+
+    with pytest.raises(urllib.error.HTTPError):
+        with traced(
+            "capability_probe.chat gpt-4.1",
+            {"contextual_orchestrator.model_group": "gpt-4.1"},
+        ):
+            raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body)
+
+    span.set_attribute.assert_any_call(
+        "contextual_orchestrator.error_summary",
+        "messages must mention json when response_format is json_object",
+    )
+    assert "provider_status=400" in caplog.text
+    assert "model_group=gpt-4.1" in caplog.text
+    assert "AzureException" not in caplog.text
+    assert "No fallback model group" not in caplog.text
+    assert "customer-private-text" not in caplog.text
+    assert "private.example" not in caplog.text
+
+
+def test_traced_does_not_export_natural_language_provider_echo(monkeypatch, caplog):
+    """Unstructured provider prose is never evidence that request text is absent."""
+    import urllib.error
+
+    tracer = MagicMock()
+    span = tracer.start_as_current_span.return_value.__enter__.return_value
+    monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer)
+    echoed = "The supplied phrase customer-private-text is not valid JSON"
+    body = io.BytesIO(json.dumps({"error": {"message": echoed}}).encode())
+
+    with pytest.raises(urllib.error.HTTPError):
+        with traced("capability_probe.chat model-x"):
+            raise urllib.error.HTTPError("https://private.example", 400, "bad", None, body)
+
+    span.set_attribute.assert_any_call(
+        "contextual_orchestrator.error_summary", "invalid_request_error"
+    )
+    assert echoed not in caplog.text
+    assert "customer-private-text" not in caplog.text
+
+
+def test_traced_does_not_export_classified_provider_prose(monkeypatch, caplog):
+    """A previously classified provider error cannot bypass the summary allowlist."""
+    from contextual_orchestrator.provider_errors import ProviderUpstreamError
+
+    tracer = MagicMock()
+    span = tracer.start_as_current_span.return_value.__enter__.return_value
+    monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer)
+    echoed = "The supplied phrase customer-private-text is invalid"
+    error = ProviderUpstreamError(
+        agent_id="provider_agent",
+        model="model-x",
+        error_code="invalid_request_error",
+        message=echoed,
+        client_status=400,
+        provider_status=400,
+    )
+
+    with pytest.raises(ProviderUpstreamError):
+        with traced("chat model-x"):
+            raise error
+
+    span.set_attribute.assert_any_call(
+        "contextual_orchestrator.error_summary", "invalid_request_error"
+    )
+    assert echoed not in caplog.text
+    assert "customer-private-text" not in caplog.text
+
+
 def test_annotate_and_usage_helpers_filter_to_allowed_genai_attributes():
     """Span annotation keeps approved scalars only; prompts never enter spans."""
     span = MagicMock()