From b30553df5f381cad7756508886f859577244645d Mon Sep 17 00:00:00 2001 From: rezaho Date: Sat, 12 Sep 2026 12:48:40 +0200 Subject: [PATCH 1/2] Ask the provider why a prompt cache did or did not match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenAI Responses surface can be asked to compare a request against an earlier response and to say what it found: `comparison_response_id` inside `prompt_cache_options` on the request, `prompt_cache_diagnostics` on the reply. A caller could not use either. The kwarg was not in the parameter allow-list, so it was dropped with an unknown-parameter warning before it reached the wire, and the parser read `output` and `usage` and nothing else, so the verdict was discarded on the way back. Both halves now work, behind a gate narrower than the one on the explicit markers beside them. The markers degrade to today's behaviour where they are unsupported; this field fails the whole request. Measured on 2026-09-12: an Azure v1 Responses deployment answered `prompt_cache_options. comparison_response_id` with HTTP 400, `invalid_request_error`, code `unknown_parameter`, and returned no diagnostics object on any successful reply. So the predicate reads the resolved model family AND a first-party provider, never a deployment label — on a re-hosted surface the model name is whatever an operator typed and is not evidence about the generation underneath. No second response-id field: the Responses `resp_…` id already rides `ResponseMetadata.request_id`, which is the id a later request names as its comparison. The diagnostics object is attached only when the provider returns one, so its absence stays a fact rather than a null. Every other payload, on every other leg and on every model without the capability, is byte-identical to today's. --- src/marsys/models/adapters/openai.py | 51 +++++- .../test_openai_prompt_cache_diagnostics.py | 153 ++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tests/models/test_openai_prompt_cache_diagnostics.py diff --git a/src/marsys/models/adapters/openai.py b/src/marsys/models/adapters/openai.py index 7c16f965..95d22a15 100644 --- a/src/marsys/models/adapters/openai.py +++ b/src/marsys/models/adapters/openai.py @@ -68,6 +68,20 @@ # behind a GPT-5.6-shaped model name would otherwise receive them untested. _EXPLICIT_PROMPT_CACHE_PROVIDERS = frozenset({"openai", "azure"}) +# The narrower set for the prompt-cache DIAGNOSTICS — a caller-supplied comparison response id +# on the request, and the provider's own reason for the outcome on the reply. First-party only, +# and Azure's absence is measured rather than assumed: on 2026-09-12 an Azure v1 Responses +# deployment answered `prompt_cache_options.comparison_response_id` with HTTP 400, +# `invalid_request_error`, code `unknown_parameter`, and returned no diagnostics object on any +# reply. A request that carries the field there does not degrade — it fails — so the gate is the +# difference between a diagnostic and an outage. +_PROMPT_CACHE_DIAGNOSTICS_PROVIDERS = frozenset({"openai"}) + +# What the caller passes to ask the provider to compare this request against an earlier one, and +# where it lands inside the request's prompt-cache options. +PROMPT_CACHE_COMPARISON_KWARG = "prompt_cache_comparison_response_id" +_PROMPT_CACHE_COMPARISON_FIELD = "comparison_response_id" + # Request-level: use the request's own breakpoints instead of the provider's implicit # one on the latest message. `30m` is the default, the only accepted value and a # minimum; it is sent explicitly so the request says what it means. @@ -549,6 +563,17 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An ): payload["prompt_cache_options"] = dict(PROMPT_CACHE_OPTIONS_EXPLICIT) + # The diagnostics ask: name an earlier response and the provider says whether this + # request matched its prefix and why not. Folded into the options the request already + # carries rather than sent as a field of its own, because that is the shape the surface + # documents — and forwarded ONLY where the endpoint serves it, since the one that does + # not answers the whole request with a 400 rather than ignoring the field. + comparison = kwargs.get(PROMPT_CACHE_COMPARISON_KWARG) + if comparison and self._supports_prompt_cache_diagnostics(model_lower): + options = dict(payload.get("prompt_cache_options") or PROMPT_CACHE_OPTIONS_EXPLICIT) + options[_PROMPT_CACHE_COMPARISON_FIELD] = comparison + payload["prompt_cache_options"] = options + # Only accept known OpenAI Responses API parameters - warn about unknown ones # Based on: https://platform.openai.com/docs/api-reference/responses/create valid_openai_params = { @@ -584,6 +609,10 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An "safety_identifier", "prompt_cache_key", "prompt_cache_retention", + # Folded into `prompt_cache_options` above where the endpoint serves the + # diagnostics; named here so a caller that asks for them on a leg that does not + # is quietly ignored rather than warned about a parameter this adapter knows. + PROMPT_CACHE_COMPARISON_KWARG, "user", # Deprecated, but still accepted # Service tier "service_tier", @@ -628,6 +657,22 @@ def _supports_explicit_prompt_cache(self, model_lower: str) -> bool: return False return supports_explicit_prompt_cache(model_lower) + def _supports_prompt_cache_diagnostics(self, model_lower: str) -> bool: + """Whether this request may ask the provider to diagnose its own cache outcome. + + Narrower than the explicit markers above, and narrower in the way that matters: the + markers degrade to today's behaviour where they are unsupported, while the comparison + field takes the whole request down with a 400 on the one surface it was measured + against. So this reads the RESOLVED model family and a first-party provider, never a + deployment label — on a re-hosted surface the model name is whatever an operator typed, + so it is not evidence about the generation underneath and cannot be allowed to decide + whether a request carries a field that can fail it. + """ + provider = getattr(self, "provider", None) or self._provider_name() + if provider not in _PROMPT_CACHE_DIAGNOSTICS_PROVIDERS: + return False + return supports_explicit_prompt_cache(model_lower) + def get_endpoint_url(self) -> str: # Migrate to OpenAI Responses API (unified endpoint for all models) # Supports reasoning parameter for GPT-5, o-series, and all future models @@ -806,7 +851,10 @@ def harmonize_response( cache_creation_input_tokens=cache_write_tokens or None, ) - # Build metadata + # Build metadata. ``request_id`` already carries the Responses `resp_…` id, which is + # what a later request names as its comparison — so the diagnostics need no second + # response-id field, only the provider's own verdict beside it when one is returned. + diagnostics = raw_response.get("prompt_cache_diagnostics") metadata = ResponseMetadata( provider=self._provider_name() or "openai", model=raw_response.get("model", self.model_name), @@ -815,6 +863,7 @@ def harmonize_response( usage=usage, finish_reason=finish_reason, response_time=time.time() - request_start_time, + **({"prompt_cache_diagnostics": diagnostics} if diagnostics else {}), ) # Handle content - provide a default message if truncated diff --git a/tests/models/test_openai_prompt_cache_diagnostics.py b/tests/models/test_openai_prompt_cache_diagnostics.py new file mode 100644 index 00000000..69b3f869 --- /dev/null +++ b/tests/models/test_openai_prompt_cache_diagnostics.py @@ -0,0 +1,153 @@ +"""The prompt-cache diagnostics: asking the provider why a prefix did or did not match. + +Two halves of one question. On the REQUEST, a caller names an earlier response and the provider +compares this request's prefix against it. On the REPLY, the provider says what it found. Between +them they turn "the cache read nothing and all three of our hashes are equal" from a dead end into +a sentence somebody wrote down. + +The gate is narrower than the one on the explicit markers beside it, and the difference is +measured rather than cautious. On 2026-09-12 an Azure v1 Responses deployment answered +``prompt_cache_options.comparison_response_id`` with HTTP 400, ``invalid_request_error``, code +``unknown_parameter``, and returned no diagnostics object on any successful reply. The markers +degrade where they are unsupported; this field fails the whole request, so it goes only where the +endpoint is known to serve it. + +No network anywhere in this file. +""" + +import pytest + +from marsys.models.adapters.azure import AsyncAzureOpenAIAdapter, AzureOpenAIAdapter +from marsys.models.adapters.openai import ( + PROMPT_CACHE_COMPARISON_KWARG, + AsyncOpenAIAdapter, + OpenAIAdapter, +) + +FIRST_PARTY = [OpenAIAdapter, AsyncOpenAIAdapter] +AZURE = [AzureOpenAIAdapter, AsyncAzureOpenAIAdapter] +MESSAGES = [{"role": "user", "content": "hello"}] +RESPONSE_ID = "resp_abc123" + + +def _make(adapter_type, model_name="gpt-5.6-terra"): + return adapter_type( + model_name=model_name, api_key="not-a-real-key", + base_url="https://example.invalid/openai/v1", max_tokens=1024, + ) + + +def _options(payload): + return payload.get("prompt_cache_options") or {} + + +# --- the request side --------------------------------------------------------------- + + +@pytest.mark.parametrize("adapter_type", FIRST_PARTY) +def test_the_comparison_id_reaches_the_wire_inside_the_prompt_cache_options(adapter_type): + payload = _make(adapter_type).format_request_payload( + MESSAGES, **{PROMPT_CACHE_COMPARISON_KWARG: RESPONSE_ID} + ) + assert _options(payload)["comparison_response_id"] == RESPONSE_ID + # …inside the options the request already carries, not beside them + assert "comparison_response_id" not in payload + assert _options(payload)["mode"] == "explicit" + + +@pytest.mark.parametrize("adapter_type", FIRST_PARTY) +def test_a_caller_that_asks_for_nothing_sends_no_comparison(adapter_type): + payload = _make(adapter_type).format_request_payload(MESSAGES) + assert "comparison_response_id" not in _options(payload) + + +@pytest.mark.parametrize("adapter_type", AZURE) +def test_azure_never_sends_the_field_however_the_caller_asks(adapter_type): + """The measured 400. A request that carries it there does not degrade, it fails.""" + payload = _make(adapter_type).format_request_payload( + MESSAGES, **{PROMPT_CACHE_COMPARISON_KWARG: RESPONSE_ID} + ) + assert "comparison_response_id" not in _options(payload) + assert _options(payload).get("mode") == "explicit" # …and the markers are untouched + + +@pytest.mark.parametrize("adapter_type", FIRST_PARTY) +def test_a_model_before_the_serving_generation_never_sends_the_field(adapter_type): + payload = _make(adapter_type, "gpt-5.5").format_request_payload( + MESSAGES, **{PROMPT_CACHE_COMPARISON_KWARG: RESPONSE_ID} + ) + assert "prompt_cache_options" not in payload + + +def test_the_capability_reads_the_resolved_family_and_the_provider_not_a_label(): + """Two deployments with different labels resolving to the same family must behave the same — + so the predicate cannot be allowed to read the label. On a re-hosted surface the name is + whatever an operator typed, which is why that whole provider is out rather than its names + being sorted one by one.""" + first_party = _make(OpenAIAdapter) + assert first_party._supports_prompt_cache_diagnostics("gpt-5.6-terra") + assert first_party._supports_prompt_cache_diagnostics("gpt-6") + assert not first_party._supports_prompt_cache_diagnostics("gpt-5.5") + + azure = _make(AzureOpenAIAdapter) + for label in ("gpt-5.6-terra", "my-deployment", "gpt-6"): + assert not azure._supports_prompt_cache_diagnostics(label) + + +@pytest.mark.parametrize("adapter_type", FIRST_PARTY + AZURE) +def test_every_other_payload_is_byte_identical_to_one_built_without_the_kwarg(adapter_type): + adapter = _make(adapter_type) + plain = adapter.format_request_payload(MESSAGES, temperature=0.5) + asked = adapter.format_request_payload( + MESSAGES, temperature=0.5, **{PROMPT_CACHE_COMPARISON_KWARG: None} + ) + assert plain == asked + + +@pytest.mark.parametrize("adapter_type", FIRST_PARTY + AZURE) +def test_the_kwarg_is_a_known_parameter_and_raises_no_unknown_parameter_warning(adapter_type): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error") + _make(adapter_type).format_request_payload( + MESSAGES, **{PROMPT_CACHE_COMPARISON_KWARG: RESPONSE_ID} + ) + + +# --- the reply side ----------------------------------------------------------------- + + +def _reply(**extra): + return { + "id": RESPONSE_ID, + "model": "gpt-5.6-terra", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "ok"}]}], + "usage": {"input_tokens": 10, "output_tokens": 2}, + **extra, + } + + +def test_the_provider_s_diagnostics_ride_the_harmonized_response_when_it_returns_them(): + adapter = _make(OpenAIAdapter) + response = adapter.harmonize_response( + _reply(prompt_cache_diagnostics={"reason": "input_changed"}), request_start_time=0.0 + ) + assert response.metadata.prompt_cache_diagnostics == {"reason": "input_changed"} + + +def test_they_are_absent_rather_than_null_when_the_provider_returns_none(): + adapter = _make(OpenAIAdapter) + response = adapter.harmonize_response(_reply(), request_start_time=0.0) + assert not hasattr(response.metadata, "prompt_cache_diagnostics") + + +def test_no_second_response_id_field_is_added(): + """The Responses id already rides ``request_id``, and that is the id a later request names as + its comparison. A second field for one fact is two places for it to be wrong.""" + adapter = _make(OpenAIAdapter) + response = adapter.harmonize_response( + _reply(prompt_cache_diagnostics={"reason": "cache_hit"}), request_start_time=0.0 + ) + assert response.metadata.request_id == RESPONSE_ID + assert not hasattr(response.metadata, "response_id") From 54909cbc8eb751fcbb988555b4e48cf103d10fbe Mon Sep 17 00:00:00 2001 From: rezaho Date: Sat, 12 Sep 2026 13:45:59 +0200 Subject: [PATCH 2/2] Say what the diagnostics gate actually reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The predicate's docstring claimed a resolved model family. Nothing resolves one at request time: the generation comes off the model name through the same regex the explicit markers use, and on a re-hosted surface that name is whatever an operator typed on the deployment. What keeps a label out of the decision is the provider set, which admits first-party endpoints alone — and there the name is the model, so no deployment label is ever consulted. The docstring and the test that pins it now say that, because the next reader who widens the set on the strength of a family read would re-earn the measured 400. --- src/marsys/models/adapters/openai.py | 13 +++++++++---- .../models/test_openai_prompt_cache_diagnostics.py | 11 ++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/marsys/models/adapters/openai.py b/src/marsys/models/adapters/openai.py index 95d22a15..12b7cd16 100644 --- a/src/marsys/models/adapters/openai.py +++ b/src/marsys/models/adapters/openai.py @@ -663,10 +663,15 @@ def _supports_prompt_cache_diagnostics(self, model_lower: str) -> bool: Narrower than the explicit markers above, and narrower in the way that matters: the markers degrade to today's behaviour where they are unsupported, while the comparison field takes the whole request down with a 400 on the one surface it was measured - against. So this reads the RESOLVED model family and a first-party provider, never a - deployment label — on a re-hosted surface the model name is whatever an operator typed, - so it is not evidence about the generation underneath and cannot be allowed to decide - whether a request carries a field that can fail it. + against. + + The PROVIDER set is the guarantee, and it is what carries the whole weight here. Nothing + resolves a model family at request time: the generation is read off the model name by the + same regex the markers use, and on a re-hosted surface that name is whatever an operator + typed on the deployment. What keeps such a label out of this decision is that the set + admits first-party endpoints alone, where the name IS the model — so no deployment label + is ever consulted. Widen the set and the name check stops being sound, which is the 400 + this gate was measured to avoid. """ provider = getattr(self, "provider", None) or self._provider_name() if provider not in _PROMPT_CACHE_DIAGNOSTICS_PROVIDERS: diff --git a/tests/models/test_openai_prompt_cache_diagnostics.py b/tests/models/test_openai_prompt_cache_diagnostics.py index 69b3f869..e920b063 100644 --- a/tests/models/test_openai_prompt_cache_diagnostics.py +++ b/tests/models/test_openai_prompt_cache_diagnostics.py @@ -79,11 +79,12 @@ def test_a_model_before_the_serving_generation_never_sends_the_field(adapter_typ assert "prompt_cache_options" not in payload -def test_the_capability_reads_the_resolved_family_and_the_provider_not_a_label(): - """Two deployments with different labels resolving to the same family must behave the same — - so the predicate cannot be allowed to read the label. On a re-hosted surface the name is - whatever an operator typed, which is why that whole provider is out rather than its names - being sorted one by one.""" +def test_the_provider_set_is_what_keeps_a_deployment_label_out_of_the_capability(): + """The gate is the first-party provider set plus the generation read from the model name. On a + first-party endpoint the name IS the model, so no deployment label is ever consulted; on a + re-hosted surface the name is whatever an operator typed, which is why that whole provider is + out rather than its names being sorted one by one. Nothing resolves a family at request time, + and nothing needs to — widen the set and the name check stops being sound.""" first_party = _make(OpenAIAdapter) assert first_party._supports_prompt_cache_diagnostics("gpt-5.6-terra") assert first_party._supports_prompt_cache_diagnostics("gpt-6")