From 1c8a438c9222cccf300f00929753bc6956968bee Mon Sep 17 00:00:00 2001 From: rezaho Date: Fri, 11 Sep 2026 15:51:09 +0200 Subject: [PATCH] Place explicit prompt-cache breakpoints on the OpenAI-family legs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the Responses legs the cache never covered the conversation body. The provider's default mode puts its breakpoint on the latest message, and this stack's latest message is a status row carrying a clock that changes every call, so the entry each request wrote ended inside bytes the next request no longer sent. Measured on gpt-5.6-terra over three growing eight-round requests: every step read about 1,642 tokens, the system text and the tool list and none of the body, and rewrote nine and then ten thousand tokens of byte-identical conversation at the 1.25x cache-write premium. The payload builder now asks for explicit mode and puts a breakpoint on every durable input item. Marking every item rather than the tail is what makes the rule work: a breakpoint is matched only while it is still present in the request being sent, so a single marker moved forward one row per request leaves nothing behind and reads zero — measured, and it cost more than sending nothing. Marking every item makes the marked set a function of each row's own position from the start of the list, so the row that was last on the previous request is marked by construction and the rule needs no memory. The same three requests then read 10,620 and 12,109 of a 13,717-token prompt, with a 1,489-token write per step. Two gates gate it. The model, because anything before the GPT-5.6 family answers either field with a 400. The provider, because the factory routes every unrecognized provider to this adapter and a third-party endpoint behind a 5.6-shaped name would otherwise receive fields only OpenAI's and Azure's surfaces have been measured to take; for that check the factory now stamps the requested provider on each adapter it builds, which is the only layer that knows it. prompt_cache_options is set by the branch that placed the marker and only then: explicit mode with no breakpoint is the documented way to turn caching off, and measures as exactly that. A caller's contribution stays one flag. CACHE_EXEMPT_KEY moves to base now that both families read it, re-exported from anthropic where it was defined. The Anthropic legs keep their tail rule, the OAuth twin is untouched — it speaks to a backend neither provider page covers — and payloads for every pre-5.6 model, both OAuth legs, both Anthropic legs and Bedrock are byte-identical to before, checked by rebuilding them from the previous commit and diffing. The one wire-shape change: a tool result's output stops being a string and becomes a one-part input_text list, because a breakpoint can only ride a content block. The text the model reads is unchanged. --- src/marsys/models/adapters/anthropic.py | 22 +- src/marsys/models/adapters/azure.py | 24 +- src/marsys/models/adapters/base.py | 24 + src/marsys/models/adapters/factory.py | 12 +- src/marsys/models/adapters/openai.py | 309 +++++++++-- src/marsys/models/models.py | 7 +- tests/models/test_azure_openai_leg.py | 30 +- tests/models/test_openai_prompt_cache.py | 535 +++++++++++++++++++ tests/models/test_prompt_cache_breakpoint.py | 69 +++ tests/models/test_prompt_cache_key.py | 44 -- 10 files changed, 949 insertions(+), 127 deletions(-) create mode 100644 tests/models/test_openai_prompt_cache.py delete mode 100644 tests/models/test_prompt_cache_key.py diff --git a/src/marsys/models/adapters/anthropic.py b/src/marsys/models/adapters/anthropic.py index eed52c06..baceca88 100644 --- a/src/marsys/models/adapters/anthropic.py +++ b/src/marsys/models/adapters/anthropic.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, List, Optional from marsys.models.adapters.base import ( + CACHE_EXEMPT_KEY, APIProviderAdapter, AsyncBaseAPIAdapter, _CapturedErrorResponse, @@ -86,21 +87,14 @@ def _anthropic_model_requires_adaptive_thinking(model_name: str) -> bool: CACHE_CONTROL_EPHEMERAL = {"type": "ephemeral"} -# A caller marks a message row with this key to say "my content here changes every -# request; do not put the cache breakpoint on me". Neutral and per-item, riding the -# caller's own message dict — the `defer_loading` shape, which is this codebase's -# established way for a caller to signal request structure without a new request -# parameter. Stripped during conversion; it never reaches the wire. +# ``CACHE_EXEMPT_KEY`` — a caller marks a message row with this key to say "my content +# here changes every request; do not put the cache breakpoint on me". Neutral and +# per-item, riding the caller's own message dict — the `defer_loading` shape, which is +# this codebase's established way for a caller to signal request structure without a new +# request parameter. Stripped during conversion; it never reaches the wire. # -# Why a caller needs this: the breakpoint's value is that the NEXT request can read -# the entry this one writes, which requires the entry's hashed prefix to consist of -# bytes the next request still contains. A row whose text is regenerated per request -# (a clock, a budget figure, anything derived from "now") is by construction absent -# from the next request, so an entry written at or after it is unreadable forever — -# each turn writes a fresh entry and reads none. Measured on Bedrock/Opus 5, single- -# step turns, tools present: marker on the volatile row → turn 2 `read=0`; marker on -# the last durable row → turn 2 `read=8425`. -CACHE_EXEMPT_KEY = "cache_exempt" +# It lives in ``base`` now that both adapter families honour it, and is re-exported here +# because this module is where it was first defined and where callers import it from. def mark_conversation_tail_for_cache( diff --git a/src/marsys/models/adapters/azure.py b/src/marsys/models/adapters/azure.py index 2911c76b..ee9f8877 100644 --- a/src/marsys/models/adapters/azure.py +++ b/src/marsys/models/adapters/azure.py @@ -38,13 +38,23 @@ Two behaviours of this endpoint that are inherited rather than worked around, recorded because both are silent until they are not: -* **Prompt caching is implicit and takes no request parameter.** ``prompt_cache_options`` - accepts ``implicit`` (the default) and ``explicit``; a ``prompt_cache_breakpoint`` - field is rejected as an unknown parameter in every position, and ``explicit`` mode - without one caches nothing. So the correct request is one that says nothing about - caching, and a measured pair of identical large calls reports - ``cache_write_tokens: 3395`` then ``cached_tokens: 3395`` with no parameter sent. - The inherited harmonizer reads both figures. +* **Prompt cache breakpoints ride inside a content block, and the inherited builder + places them.** ``prompt_cache_breakpoint`` belongs on an ``input_text``, + ``input_image`` or ``input_file`` block — including the content parts of a + ``function_call_output`` — never at item level, never at request level, and never on + the top-level ``instructions`` field. An earlier reading of this surface recorded the + field as rejected in every position and concluded that the correct request says + nothing about caching; twenty live requests on ``gpt-5.6-terra`` refuted that, all + 200, with the field inside content blocks. The limits that do bind: a request creates + at most four new cache writes (in explicit mode, its latest four breakpoints, and a + breakpoint covers everything before it), reads consider at most the latest fifty + breakpoints, the cacheable prefix is at least 1,024 tokens, breakpoints are served on + Standard pay-as-you-go deployments and silently not on PTU-M, and models before the + GPT-5.6 family answer either field with a 400 — which is why the inherited builder + gates both on the model name. ``explicit`` mode with no breakpoint is the documented + way to turn caching off and measures as exactly that: nothing written, nothing read, + the whole prompt at plain input price. The inherited harmonizer reads back both the + cached and the written figures. * **Reasoning-capable deployments reject ``temperature``.** The inherited capability check is a regex over the model name, which the real deployment names (``gpt-5.6-*``) satisfy. A deployment renamed to something not starting ``gpt-5``+ diff --git a/src/marsys/models/adapters/base.py b/src/marsys/models/adapters/base.py index 7d1efc1f..1c61a56f 100644 --- a/src/marsys/models/adapters/base.py +++ b/src/marsys/models/adapters/base.py @@ -22,6 +22,30 @@ logger = logging.getLogger(__name__) +# A caller's per-request row: content this request renders fresh (a clock, a running +# budget figure, anything derived from "now"), which the NEXT request will not contain +# verbatim. Set it on a message dict and every payload builder that places prompt-cache +# breakpoints will keep its markers off that row. +# +# Why a caller needs to say this and why it is the ONLY thing a caller says about +# caching: a breakpoint is worth something because the next request can read the entry +# this one writes, which requires the entry's hashed prefix to be bytes the next request +# still sends. An entry written at or after a regenerated row is unreadable forever — +# every request writes a fresh entry and reads none. Which rows may carry a marker, +# which block types accept one, and whether a marker may move are provider facts the +# payload builder knows and the caller does not, so placement stays adapter-side and +# this flag stays the whole interface. Measured on Bedrock/Opus 5, single-step turns, +# tools present: marker on the volatile row → turn 2 `read=0`; marker on the last +# durable row → turn 2 `read=8425`. Measured on Azure/gpt-5.6-terra, eight-round turns: +# markers left on every durable row → request 3 reads 12,109 of 13,717; the same single +# marker moved forward each request → reads 0 on every request. +# +# Family-neutral by residence: the Anthropic family reads it to find the last durable +# row, the OpenAI family reads it to skip a row entirely. Re-exported from +# ``marsys.models.adapters.anthropic``, where it was first defined. +CACHE_EXEMPT_KEY = "cache_exempt" + + # Hardcoded fallbacks used when no ErrorHandlingConfig is provided. Match the # pre-Phase-1 behaviour exactly so existing tests and ad-hoc adapter usage # (without an ExecutionConfig) keep working. diff --git a/src/marsys/models/adapters/factory.py b/src/marsys/models/adapters/factory.py index 282990e6..acc87be0 100644 --- a/src/marsys/models/adapters/factory.py +++ b/src/marsys/models/adapters/factory.py @@ -43,9 +43,17 @@ def create_adapter( # OAuth providers don't use api_key/base_url - they load credentials from CLI if provider in ("openai-oauth", "anthropic-oauth"): - return adapter_class(model_name, **kwargs) + adapter = adapter_class(model_name, **kwargs) + else: + adapter = adapter_class(model_name, api_key, base_url, **kwargs) - return adapter_class(model_name, api_key, base_url, **kwargs) + # The requested provider, stamped here because this is the only layer that + # knows it: several providers share one adapter class, and the fallback above + # hands every unrecognized provider to the OpenAI class outright, so a class + # name is not evidence of which endpoint a request is bound for. Adapters that + # gate an endpoint-specific request field on provider identity read this. + adapter.provider = provider + return adapter class LocalAdapterFactory: diff --git a/src/marsys/models/adapters/openai.py b/src/marsys/models/adapters/openai.py index bdcb3800..7c16f965 100644 --- a/src/marsys/models/adapters/openai.py +++ b/src/marsys/models/adapters/openai.py @@ -1,10 +1,12 @@ import json import logging +import re import time import warnings -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Collection, Dict, List, Optional from marsys.models.adapters.base import ( + CACHE_EXEMPT_KEY, APIProviderAdapter, AsyncBaseAPIAdapter, _CapturedErrorResponse, @@ -54,6 +56,223 @@ }) +# --- prompt caching ----------------------------------------------------------------- +# +# The generation that serves explicit prompt caching. Earlier models answer +# `prompt_cache_options` or `prompt_cache_breakpoint` with a 400, so the fields are +# gated on the model name rather than sent everywhere. +_EXPLICIT_PROMPT_CACHE_MIN_VERSION = (5, 6) + +# The providers whose endpoints were measured to serve these fields. The factory routes +# an unrecognized provider to this adapter, so a third-party OpenAI-compatible endpoint +# behind a GPT-5.6-shaped model name would otherwise receive them untested. +_EXPLICIT_PROMPT_CACHE_PROVIDERS = frozenset({"openai", "azure"}) + +# Request-level: use the request's own breakpoints instead of the provider's implicit +# one on the latest message. `30m` is the default, the only accepted value and a +# minimum; it is sent explicitly so the request says what it means. +PROMPT_CACHE_OPTIONS_EXPLICIT = {"mode": "explicit", "ttl": "30m"} + +# Block-level: the cacheable prefix ends at the end of the block carrying this. +PROMPT_CACHE_BREAKPOINT_EXPLICIT = {"mode": "explicit"} + +# The Responses content blocks that accept a breakpoint. An assistant/output block does +# not, and neither does the request-level `instructions` field. +_BREAKPOINT_BLOCK_TYPES = frozenset({"input_text", "input_image", "input_file"}) + +# The input-message roles whose items may carry one. `assistant` is excluded: its items +# are model output replayed back, and the provider takes a breakpoint only on input +# content. +_BREAKPOINT_ITEM_ROLES = frozenset({"system", "developer", "user"}) + +_GENERATION_RE = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") + + +def supports_explicit_prompt_cache(model_lower: str) -> bool: + """Whether a model name is GPT-5.6 or later, the generation that serves the fields. + + Reads the name the same way the temperature rule does (`format_request_payload`), + and carries the same honesty caveat: on Azure this is an operator-chosen deployment + label, so a deployment named after a model it does not serve lies to this check. + The two directions fail differently and both are acceptable. A pre-5.6 model behind + a 5.6-shaped name takes a 400 on its first call — loud, immediate, and impossible to + mistake for a cost problem. A 5.6 model behind an older-shaped name simply keeps + today's behaviour and pays today's price. + """ + match = _GENERATION_RE.match(model_lower or "") + if not match: + return False + major = int(match.group(1)) + minor = int(match.group(2) or 0) + return (major, minor) >= _EXPLICIT_PROMPT_CACHE_MIN_VERSION + + +def _blocks_with_breakpoint( + blocks: List[Any], +) -> Optional[List[Any]]: + """A copy of ``blocks`` carrying a breakpoint on the last block that can hold one. + + ``None`` when no block can, so the caller can leave the item exactly as it arrived + rather than rewriting a shape for a marker it never placed. Copies rather than + stamping in place: the durable conversation shares these dicts (the same hazard + ``hydrate_messages`` documents), so a marker written in place would leak into + persisted rows, and copying is also what makes building a payload twice from the + same input byte-identical. + """ + for index in range(len(blocks) - 1, -1, -1): + block = blocks[index] + if not isinstance(block, dict) or block.get("type") not in _BREAKPOINT_BLOCK_TYPES: + continue + if block.get("prompt_cache_breakpoint"): + # A caller placed one already; a second would buy nothing. + return list(blocks) + marked = list(blocks) + marked[index] = { + **block, + "prompt_cache_breakpoint": dict(PROMPT_CACHE_BREAKPOINT_EXPLICIT), + } + return marked + return None + + +def _mark_item_for_prompt_cache(item: Dict[str, Any]) -> bool: + """Place one breakpoint on ``item`` in place, reporting whether it landed.""" + if item.get("type") == "function_call_output": + output = item.get("output") + if isinstance(output, str): + if not output: + # An empty result has nothing to hash; the API rejects an empty text + # block, and a bare "" is what this adapter already sends. + return False + item["output"] = [{ + "type": "input_text", + "text": output, + "prompt_cache_breakpoint": dict(PROMPT_CACHE_BREAKPOINT_EXPLICIT), + }] + return True + if not isinstance(output, list) or not output: + return False + # A list-form result is converted here rather than in the item builder above, + # so a model that does not serve the fields keeps receiving its blocks + # untouched: a breakpoint can only ride a Responses block, so the conversion + # exists for the marker and happens only where the marker does. + marked = _blocks_with_breakpoint(_convert_content_types(output)) + if marked is None: + return False + item["output"] = marked + return True + + if item.get("role") not in _BREAKPOINT_ITEM_ROLES: + return False + content = item.get("content") + if isinstance(content, str): + if not content: + return False + item["content"] = [{ + "type": "input_text", + "text": content, + "prompt_cache_breakpoint": dict(PROMPT_CACHE_BREAKPOINT_EXPLICIT), + }] + return True + if not isinstance(content, list) or not content: + return False + marked = _blocks_with_breakpoint(content) + if marked is None: + return False + item["content"] = marked + return True + + +def mark_items_for_prompt_cache( + items: List[Dict[str, Any]], *, exempt_indices: Collection[int] = frozenset() +) -> bool: + """Breakpoint EVERY durable input item, in place on ``items``. True if any landed. + + Adapter-owned and unconditional, the same position and the same argument as this + codebase's other cache-marker helper (:func:`~marsys.models.adapters.anthropic. + mark_conversation_tail_for_cache`): only the payload builder knows the rendered + block layout, caching is prefix-match arithmetic over exactly those bytes, and a + caller that forgets silently re-pays full price on the whole conversation. + + Why EVERY item and not the tail, which is what the sibling helper does. On this + provider a breakpoint is matched only while it is still present in the request being + sent, so a single marker moved forward one row per request leaves nothing behind for + the next request to match: measured over three growing eight-round requests against + gpt-5.6-terra, a moving tail marker read 0 tokens every time and cost more than + sending nothing at all, while markers left on every durable row read 10,620 then + 12,109 of a 13,717-token prompt. Marking every item makes the set of marked rows a + function of each row's own position from the START of the list, so applying the rule + to a conversation and to that conversation plus new rows marks the same rows on the + shared prefix. That is the whole reason it works, and it is why the rule needs no + memory of what an earlier request sent. + + The provider's own limits are satisfied by construction rather than by arithmetic + here: at most four new cache writes per request (in explicit mode, the latest four + breakpoints, and a breakpoint covers everything before it, so the newest one writes + the whole new prefix), and reads consider the latest fifty breakpoints (the newest + marker is always within a handful of the end). + + ``exempt_indices`` are the positions the caller flagged ``CACHE_EXEMPT_KEY`` — rows + it regenerates per request. They are skipped wherever they sit, not just at the + tail, and keep the shape they arrived in. + """ + placed = False + for index, item in enumerate(items): + if index in exempt_indices: + continue + if isinstance(item, dict) and _mark_item_for_prompt_cache(item): + placed = True + return placed + + +# --- content conversion ------------------------------------------------------------- + + +def _convert_content_types(content): + """Convert Chat Completions content types to Responses API content types. + + Chat Completions format: + - {"type": "text", "text": "..."} + - {"type": "image_url", "image_url": {"url": "..."}} + + Responses API format: + - {"type": "input_text", "text": "..."} + - {"type": "input_image", "image_url": "..."} + """ + if isinstance(content, str): + return content + if isinstance(content, list): + converted = [] + for item in content: + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "text": + # Convert "text" -> "input_text" + converted.append({ + "type": "input_text", + "text": item.get("text", "") + }) + elif item_type == "image_url": + # Convert "image_url" -> "input_image" + # Also flatten: {"image_url": {"url": "..."}} -> {"image_url": "..."} + image_url_data = item.get("image_url", {}) + if isinstance(image_url_data, dict): + url = image_url_data.get("url", "") + else: + url = image_url_data + converted.append({ + "type": "input_image", + "image_url": url + }) + else: + # Keep other types as-is (input_text, input_image already correct) + converted.append(item) + else: + converted.append(item) + return converted + return content + + def served_reasoning_effort(effort: str, model_lower: str) -> str: """Keep requested reasoning positive when a model does not serve minimal.""" if effort == "minimal" and ( @@ -113,8 +332,6 @@ def get_headers(self) -> Dict[str, str]: } def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, Any]: - import re - # Check if this is a reasoning model (GPT-5+, o-series) that doesn't support temperature # Based on: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning # Future-proof: Supports GPT-5.x, GPT-6+, GPT-10+, o1, o2, o10+, etc. @@ -127,53 +344,16 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An # Convert Chat Completions format messages to Responses API format # The Responses API uses a different schema for tool calls and tool responses - def convert_content_types(content): - """Convert Chat Completions content types to Responses API content types. - - Chat Completions format: - - {"type": "text", "text": "..."} - - {"type": "image_url", "image_url": {"url": "..."}} - - Responses API format: - - {"type": "input_text", "text": "..."} - - {"type": "input_image", "image_url": "..."} - """ - if isinstance(content, str): - return content - if isinstance(content, list): - converted = [] - for item in content: - if isinstance(item, dict): - item_type = item.get("type") - if item_type == "text": - # Convert "text" -> "input_text" - converted.append({ - "type": "input_text", - "text": item.get("text", "") - }) - elif item_type == "image_url": - # Convert "image_url" -> "input_image" - # Also flatten: {"image_url": {"url": "..."}} -> {"image_url": "..."} - image_url_data = item.get("image_url", {}) - if isinstance(image_url_data, dict): - url = image_url_data.get("url", "") - else: - url = image_url_data - converted.append({ - "type": "input_image", - "image_url": url - }) - else: - # Keep other types as-is (input_text, input_image already correct) - converted.append(item) - else: - converted.append(item) - return converted - return content - converted_messages = [] + # Positions of the rows the caller flagged ``CACHE_EXEMPT_KEY``, recorded here + # because the allow-list rebuild below drops the flag before anything downstream + # could read it. A source message can expand into several items (an assistant + # turn with tool calls), so the positions are collected as items are appended + # rather than counted afterwards. + cache_exempt_indices = set() for msg in messages: role = msg.get("role") + item_start = len(converted_messages) # Handle assistant messages with tool_calls -> function_call items if role == "assistant" and msg.get("tool_calls"): @@ -182,7 +362,7 @@ def convert_content_types(content): if content: converted_messages.append({ "role": "assistant", - "content": convert_content_types(content) + "content": _convert_content_types(content) }) # Convert each tool_call to a function_call item for tc in msg["tool_calls"]: @@ -231,9 +411,12 @@ def convert_content_types(content): cleaned_msg["content"] = "" else: # Convert content types (text -> input_text, image_url -> input_image) - cleaned_msg["content"] = convert_content_types(cleaned_msg["content"]) + cleaned_msg["content"] = _convert_content_types(cleaned_msg["content"]) converted_messages.append(cleaned_msg) + if msg.get(CACHE_EXEMPT_KEY): + cache_exempt_indices.update(range(item_start, len(converted_messages))) + payload = { "model": self.model_name, "input": converted_messages, # Changed from 'messages' to 'input' for Responses API @@ -353,6 +536,19 @@ def convert_content_types(content): if kwargs.get("prompt_cache_key") is not None: payload["prompt_cache_key"] = kwargs["prompt_cache_key"] + # Prompt-cache breakpoints. LAST over the input items, after every content + # conversion above, so "the last block of an item" means the block actually + # being sent. Mode and placement are one decision made in one place: explicit + # mode with no breakpoint is the documented way to turn caching OFF (measured: + # zero written, zero read, the whole prompt at plain input price), so the + # request-level option is set only when a breakpoint was actually placed and a + # narrowed placement rule can never silently disable caching. + if self._supports_explicit_prompt_cache(model_lower): + if mark_items_for_prompt_cache( + converted_messages, exempt_indices=cache_exempt_indices + ): + payload["prompt_cache_options"] = dict(PROMPT_CACHE_OPTIONS_EXPLICIT) + # Only accept known OpenAI Responses API parameters - warn about unknown ones # Based on: https://platform.openai.com/docs/api-reference/responses/create valid_openai_params = { @@ -417,6 +613,21 @@ def _served_effort(self, effort: str, model_lower: str) -> str: """Allow re-hosted surfaces to override the shared model compatibility rule.""" return served_reasoning_effort(effort, model_lower) + def _supports_explicit_prompt_cache(self, model_lower: str) -> bool: + """Whether this request may carry the explicit prompt-cache fields. + + Two gates, and the provider one is not redundant with the model one. The + factory routes every unrecognized provider to this class, so an + OpenAI-compatible third-party endpoint serving a GPT-5.6-shaped name would + otherwise receive fields only OpenAI's and Azure's surfaces have been measured + to accept. The adapter names itself the way the rest of the base class does: + the provider a model layer stamped on it, falling back to its own class name. + """ + provider = getattr(self, "provider", None) or self._provider_name() + if provider not in _EXPLICIT_PROMPT_CACHE_PROVIDERS: + return False + return supports_explicit_prompt_cache(model_lower) + def get_endpoint_url(self) -> str: # Migrate to OpenAI Responses API (unified endpoint for all models) # Supports reasoning parameter for GPT-5, o-series, and all future models diff --git a/src/marsys/models/models.py b/src/marsys/models/models.py index fcf827e2..d1250307 100644 --- a/src/marsys/models/models.py +++ b/src/marsys/models/models.py @@ -642,10 +642,9 @@ def __init__( reasoning_effort=reasoning_effort, **kwargs, ) - # The adapter emits the trace event and reports the provider in error - # fallbacks, but adapters don't otherwise carry ``provider`` — set it. - if self.adapter is not None: - self.adapter.provider = provider + # ``self.adapter.provider`` is stamped by the factory, which is the layer that + # knows which provider was asked for — several providers share one adapter + # class, so the class alone cannot say. # Try to create async adapter if available self.async_adapter = None diff --git a/tests/models/test_azure_openai_leg.py b/tests/models/test_azure_openai_leg.py index 7612b974..60b7fa71 100644 --- a/tests/models/test_azure_openai_leg.py +++ b/tests/models/test_azure_openai_leg.py @@ -271,15 +271,31 @@ def test_the_sanitize_is_the_shared_openai_behaviour_not_an_azure_special_case() assert set(payload["input"][0]) == {"role", "content"} -def test_no_cache_option_field_is_sent(): - """Prompt caching on this API is implicit and takes no request parameter. A - `prompt_cache_breakpoint` is rejected outright in every position, and `explicit` - mode without one caches nothing — so the correct request says nothing at all, - and the measured pair of identical calls still reports a write then a read.""" - payload = _azure().format_request_payload(MESSAGES) +@pytest.mark.parametrize("model", ["gpt-5", "gpt-5.4-mini", "gpt-5.5"]) +def test_a_deployment_before_the_5_6_generation_is_told_nothing_about_caching(model): + """Models before the GPT-5.6 family answer either cache field with a 400, so the + inherited builder gates both on the deployment name. An earlier reading of this + surface had the fields rejected in every position and concluded that the correct + request says nothing at all; twenty live requests on a 5.6 deployment refuted the + general claim, and this is the part of it that survived.""" + payload = _azure(model_name=model).format_request_payload(MESSAGES) assert "prompt_cache_options" not in payload - assert "prompt_cache_breakpoint" not in payload assert not any("cache" in key for key in payload) + assert "prompt_cache_breakpoint" not in str(payload) + + +@pytest.mark.parametrize("model", ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) +def test_a_5_6_deployment_asks_for_explicit_mode_and_carries_a_breakpoint(model): + """The control for the gate above, on the deployment names this fleet actually + runs. The breakpoint rides inside a content block, which is the placement the live + requests accepted; at item level or request level it is an unknown parameter.""" + payload = _azure(model_name=model).format_request_payload(MESSAGES) + assert payload["prompt_cache_options"] == {"mode": "explicit", "ttl": "30m"} + assert payload["input"][0]["content"] == [{ + "type": "input_text", + "text": "hi", + "prompt_cache_breakpoint": {"mode": "explicit"}, + }] # --- the thinking knob ------------------------------------------------------- diff --git a/tests/models/test_openai_prompt_cache.py b/tests/models/test_openai_prompt_cache.py new file mode 100644 index 00000000..462dd304 --- /dev/null +++ b/tests/models/test_openai_prompt_cache.py @@ -0,0 +1,535 @@ +"""The OpenAI-family prompt-cache contract: the caller's routing key and the markers. + +Two things ride the same payload builder and are pinned together here because they are +one feature from the caller's side — "make the cache work for this conversation": + +* ``prompt_cache_key``, the caller-selected routing key, passed through untouched. +* ``prompt_cache_options`` and ``prompt_cache_breakpoint``, placed by the adapter on + every durable input item so that within a turn each step reads the whole of the + previous step's prompt instead of rewriting it. + +The placement rule is adapter-owned and memoryless, and both halves of that matter. +Measured against gpt-5.6-terra over three growing eight-round requests: a single marker +moved forward one row per request read 0 tokens on every request and cost more than +sending nothing, while markers left on every durable row read 10,620 then 12,109 of a +13,717-token prompt. So the marked set is a function of each row's own position from the +start of the list, never a window measured from the end. + +The fields are gated on TWO facts about a request, and both gates are load-bearing. The +model: anything before the GPT-5.6 family answers either field with a 400. The provider: +the factory hands every unrecognized provider to the OpenAI adapter, so a third-party +OpenAI-compatible endpoint behind a 5.6-shaped model name would otherwise receive fields +only OpenAI's and Azure's surfaces are known to take. + +No network anywhere in this file. +""" + +import json + +import pytest + +from marsys.models.adapters.anthropic import CACHE_EXEMPT_KEY as ANTHROPIC_CACHE_EXEMPT_KEY +from marsys.models.adapters.azure import AsyncAzureOpenAIAdapter, AzureOpenAIAdapter +from marsys.models.adapters.base import CACHE_EXEMPT_KEY +from marsys.models.adapters.factory import ProviderAdapterFactory +from marsys.models.adapters.openai import AsyncOpenAIAdapter, OpenAIAdapter +from marsys.models.adapters.openai_oauth import AsyncOpenAIOAuthAdapter, OpenAIOAuthAdapter + +BREAKPOINT = {"mode": "explicit"} +OPTIONS = {"mode": "explicit", "ttl": "30m"} + +# Names the fields are sent for, and names they are not. `gpt-5.5` is the last release +# before the generation that serves them; `gpt-5` sorts before `gpt-5.6` only if the +# minor version is read as a number rather than a string. +SUPPORTING = ["gpt-5.6", "gpt-5.6-terra", "gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.7", "gpt-6", "gpt-10-mini"] +NON_SUPPORTING = ["gpt-5", "gpt-5.4-mini", "gpt-5.5", "gpt-4o", "o3-mini", "not-a-gpt-model"] + +# The four builders that inherit the OpenAI payload builder, and the two OAuth ones that +# do not (``OpenAIOAuthAdapter`` subclasses the base adapter directly and speaks to the +# ChatGPT backend, which neither provider page covers). +INHERITING = [OpenAIAdapter, AsyncOpenAIAdapter, AzureOpenAIAdapter, AsyncAzureOpenAIAdapter] +OAUTH = [OpenAIOAuthAdapter, AsyncOpenAIOAuthAdapter] + + +def _make(adapter_type, model_name): + if issubclass(adapter_type, OpenAIOAuthAdapter): + # Payload construction needs no credential discovery or network client. + result = object.__new__(adapter_type) + result.model_name = model_name + return result + return adapter_type( + model_name=model_name, api_key="not-a-real-key", + base_url="https://example.invalid/openai/v1", max_tokens=1024, + ) + + +@pytest.fixture(params=INHERITING + OAUTH) +def adapter_type(request): + return request.param + + +@pytest.fixture(params=["gpt-5", "gpt-5.6-terra"]) +def model_name(request): + """One name from each side of the generation gate. + + Every contract below that is about the caller's key rather than the markers has to + hold on both, because the key predates the markers and must not start depending on + them. + """ + return request.param + + +@pytest.fixture +def adapter(adapter_type, model_name): + return _make(adapter_type, model_name) + + +@pytest.fixture(params=INHERITING) +def builder(request): + """Only the adapters that inherit the OpenAI payload builder.""" + return request.param + + +# --- helpers ------------------------------------------------------------------------ + + +def _items(payload): + return payload["input"] + + +def _texts(item): + """Every text the model reads in one item, in order.""" + value = item.get("output") if item.get("type") == "function_call_output" else item.get("content") + if isinstance(value, str): + return [value] if value else [] + if isinstance(value, list): + return [b.get("text", "") for b in value if isinstance(b, dict) and "text" in b] + return [] + + +def _markers(payload): + """Every breakpoint anywhere in the payload, as (item index, block index).""" + found = [] + for i, item in enumerate(_items(payload)): + blocks = item.get("output") if item.get("type") == "function_call_output" else item.get("content") + if not isinstance(blocks, list): + continue + for j, block in enumerate(blocks): + if isinstance(block, dict) and block.get("prompt_cache_breakpoint"): + found.append((i, j)) + return found + + +def _collapse(value): + """A one-block ``input_text`` list reads back as the string it was promoted from.""" + if isinstance(value, list) and len(value) == 1 and isinstance(value[0], dict): + block = value[0] + if set(block) == {"type", "text"} and block["type"] == "input_text": + return block["text"] + return value + + +def _without_markers(payload): + """The payload as it would have been built with no cache marker rule at all.""" + stripped = {k: v for k, v in payload.items() if k != "prompt_cache_options"} + items = [] + for item in _items(payload): + item = dict(item) + for key in ("content", "output"): + value = item.get(key) + if isinstance(value, list): + item[key] = _collapse([ + {k: v for k, v in b.items() if k != "prompt_cache_breakpoint"} + if isinstance(b, dict) else b + for b in value + ]) + items.append(item) + stripped["input"] = items + return stripped + + +# A conversation with one of every markable shape, and the rows that must stay bare. +CONVERSATION = [ + {"role": "system", "content": "You are Spren, the founder's employee."}, + {"role": "user", "content": "Book the room."}, + { + "role": "assistant", + "content": "on it", + "tool_calls": [{"id": "call_1", "function": {"name": "search", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "three rooms free"}, + {"role": "assistant", "content": "Booked."}, + {"role": "user", "content": "now 09:00 | spend 1.20", CACHE_EXEMPT_KEY: True}, +] + + +# --- the caller's routing key (unchanged by this session) ---------------------------- + + +def test_caller_key_reaches_payload_and_stays_stable(adapter): + messages = [{"role": "user", "content": "Continue the work."}] + key = "installation:instance" + first = adapter.format_request_payload(messages, prompt_cache_key=key) + second = adapter.format_request_payload(messages, prompt_cache_key=key) + assert first["prompt_cache_key"] == second["prompt_cache_key"] == key + other = adapter.format_request_payload(messages, prompt_cache_key="installation:other") + assert other["prompt_cache_key"] == "installation:other" + + +@pytest.mark.parametrize("kwargs", [{}, {"prompt_cache_key": None}]) +def test_absent_key_is_omitted_without_minting_one(adapter, kwargs): + messages = [{"role": "user", "content": "Continue the work."}] + first = adapter.format_request_payload(messages, **kwargs) + second = adapter.format_request_payload(messages, **kwargs) + assert "prompt_cache_key" not in first + assert "prompt_cache_key" not in second + assert first == second + + +# --- AC-1: explicit mode follows the placement -------------------------------------- + + +@pytest.mark.parametrize("model", SUPPORTING) +def test_a_marked_request_asks_for_explicit_mode(builder, model): + """AC-1a.""" + payload = _make(builder, model).format_request_payload(CONVERSATION) + assert _markers(payload) + assert payload["prompt_cache_options"] == OPTIONS + + +@pytest.mark.parametrize("messages", [ + [], + [{"role": "assistant", "content": "nothing to cache here"}], + [{"role": "user", "content": "now 09:00", CACHE_EXEMPT_KEY: True}], + [{"role": "user", "content": ""}], + [{"role": "tool", "tool_call_id": "call_1", "content": ""}], +]) +def test_an_unmarked_request_sends_no_mode_at_all(builder, messages): + """AC-1b. Explicit mode with no breakpoint is the documented way to turn caching + OFF, and measures as exactly that — nothing written, nothing read, the whole prompt + at plain input price. So the option follows the placement rather than the model + name, and a narrowed placement rule can never silently disable caching.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload(messages) + assert _markers(payload) == [] + assert "prompt_cache_options" not in payload + + +@pytest.mark.parametrize("provider", ["groq", "together", "some-new-gateway"]) +def test_an_unknown_provider_routed_to_this_builder_gets_nothing(provider): + """AC-1c. ``factory.py`` hands every unrecognized provider to the OpenAI adapter.""" + adapter = ProviderAdapterFactory.create_adapter( + provider=provider, model_name="gpt-5.6-terra", + api_key="not-a-real-key", base_url="https://example.invalid/v1", + ) + assert isinstance(adapter, OpenAIAdapter) + payload = adapter.format_request_payload(CONVERSATION) + assert "prompt_cache_options" not in payload + assert "prompt_cache_breakpoint" not in json.dumps(payload) + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +def test_the_two_measured_providers_do_get_the_fields(provider): + """AC-1c's control: the scoping is a gate, not a blanket refusal.""" + adapter = ProviderAdapterFactory.create_adapter( + provider=provider, model_name="gpt-5.6-terra", + api_key="not-a-real-key", base_url="https://example.invalid/openai/v1", + ) + payload = adapter.format_request_payload(CONVERSATION) + assert payload["prompt_cache_options"] == OPTIONS + + +# --- AC-2: the generation gate ------------------------------------------------------ + + +@pytest.mark.parametrize("model", NON_SUPPORTING) +def test_a_pre_generation_model_is_told_nothing_about_caching(builder, model): + """AC-2a. These names answer either field with a 400.""" + payload = _make(builder, model).format_request_payload( + CONVERSATION, prompt_cache_key="installation:instance" + ) + assert "prompt_cache_options" not in payload + assert "prompt_cache_breakpoint" not in json.dumps(payload) + assert [k for k in payload if "cache" in k] == ["prompt_cache_key"] + + +@pytest.mark.parametrize("model", NON_SUPPORTING) +def test_a_pre_generation_payload_is_what_it_was_before_this_session(builder, model): + """AC-2b. The literal below is the payload the builder produced at 47ce23e6, the + commit this session branched from, for exactly these messages.""" + payload = _make(builder, model).format_request_payload(CONVERSATION) + assert payload["input"] == [ + {"role": "system", "content": "You are Spren, the founder's employee."}, + {"role": "user", "content": "Book the room."}, + {"role": "assistant", "content": "on it"}, + {"type": "function_call", "call_id": "call_1", "name": "search", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "three rooms free"}, + {"role": "assistant", "content": "Booked."}, + {"role": "user", "content": "now 09:00 | spend 1.20"}, + ] + + +# --- AC-3: tool results ------------------------------------------------------------- + + +def test_a_tool_result_becomes_a_one_part_list_carrying_the_marker(builder): + """AC-3a. The one structural change on the wire: ``output`` stops being a string, + because a breakpoint can only ride a content block.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + output = _items(payload)[4] + assert output["type"] == "function_call_output" + assert output["output"] == [{ + "type": "input_text", + "text": "three rooms free", + "prompt_cache_breakpoint": BREAKPOINT, + }] + + +def test_a_block_shaped_tool_result_keeps_its_blocks(builder): + """AC-3b. Converted the way user content is, marker on the last supported block.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload([ + {"role": "user", "content": "look"}, + {"role": "tool", "tool_call_id": "call_1", "content": [ + {"type": "text", "text": "a blocky result"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}}, + ]}, + ]) + assert _items(payload)[1]["output"] == [ + {"type": "input_text", "text": "a blocky result"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,AAA", + "prompt_cache_breakpoint": BREAKPOINT, + }, + ] + + +def test_an_empty_tool_result_stays_an_empty_string(builder): + """AC-3c. The API rejects an empty text block, and an empty result has no bytes to + hash anyway.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload([ + {"role": "user", "content": "go"}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + ]) + assert _items(payload)[1]["output"] == "" + + +# --- AC-4: message items ------------------------------------------------------------ + + +@pytest.mark.parametrize("role", ["system", "developer", "user"]) +def test_every_input_role_carries_the_marker_on_its_last_block(builder, role): + """AC-4a.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload([ + {"role": role, "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "last"}, + ]}, + ]) + assert _items(payload)[0]["content"] == [ + {"type": "input_text", "text": "first"}, + {"type": "input_text", "text": "last", "prompt_cache_breakpoint": BREAKPOINT}, + ] + + +def test_a_string_content_is_promoted_only_to_carry_a_marker(builder): + """AC-4b. The promotion exists for the marker; where no marker lands, nothing about + the item's shape changes — which is what the pre-generation payload above pins.""" + marked = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + assert _items(marked)[0]["content"] == [{ + "type": "input_text", + "text": "You are Spren, the founder's employee.", + "prompt_cache_breakpoint": BREAKPOINT, + }] + bare = _make(builder, "gpt-5.5").format_request_payload(CONVERSATION) + assert _items(bare)[0]["content"] == "You are Spren, the founder's employee." + + +def test_an_empty_message_is_left_exactly_as_it_arrived(builder): + """AC-4c.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload([ + {"role": "user", "content": "the durable ask"}, + {"role": "user", "content": ""}, + {"role": "user", "content": []}, + ]) + assert _items(payload)[1]["content"] == "" + assert _items(payload)[2]["content"] == [] + + +@pytest.mark.parametrize("model", SUPPORTING + NON_SUPPORTING) +def test_an_assistant_item_never_carries_a_marker(builder, model): + """AC-4d. A breakpoint rides input content; assistant items are model output + replayed back, and function_call items carry arguments rather than content.""" + payload = _make(builder, model).format_request_payload(CONVERSATION) + for index, _ in _markers(payload): + item = _items(payload)[index] + assert item.get("role") != "assistant" + assert item.get("type") != "function_call" + + +def test_the_instructions_field_never_carries_a_marker(builder): + """AC-4e. Upstream forbids a breakpoint on the top-level instructions field; this + builder never writes that field, so the guarantee is structural.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload( + CONVERSATION, instructions="a system preamble" + ) + assert "instructions" not in payload + assert payload["prompt_cache_options"] == OPTIONS + + +# --- AC-5: the caller's per-request row --------------------------------------------- + + +@pytest.mark.parametrize("position", [0, 3, 6]) +def test_an_exempt_row_is_skipped_wherever_it_sits(builder, position): + """AC-5a. The exemption is about the row, not about the tail: a caller that puts a + regenerated row in the middle of a list must not have it marked either, because the + marker would sit on bytes the next request no longer sends.""" + rows = list(CONVERSATION) + volatile = {"role": "user", "content": "now 09:00 | spend 1.20", CACHE_EXEMPT_KEY: True} + rows.remove(volatile) + rows.insert(position, volatile) + payload = _make(builder, "gpt-5.6-terra").format_request_payload(rows) + marked_texts = [t for i, _ in _markers(payload) for t in _texts(_items(payload)[i])] + assert "now 09:00 | spend 1.20" not in marked_texts + assert _markers(payload) + + +@pytest.mark.parametrize("content, expected", [ + ("now 09:00", "now 09:00"), + ([{"type": "text", "text": "now 09:00"}], [{"type": "input_text", "text": "now 09:00"}]), +]) +def test_an_exempt_row_keeps_the_shape_it_arrived_in(builder, content, expected): + """AC-5b. A string stays a string: the promotion happens only to carry a marker.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload([ + {"role": "user", "content": "the durable ask"}, + {"role": "user", "content": content, CACHE_EXEMPT_KEY: True}, + ]) + assert _items(payload)[1]["content"] == expected + + +def test_the_exempt_flag_never_reaches_the_wire(builder): + """AC-5c and AC-5d.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + assert CACHE_EXEMPT_KEY not in json.dumps(payload) + assert _items(payload)[-1]["role"] == "user" + + +# --- AC-6: the model reads the same text -------------------------------------------- + + +def test_the_text_roles_and_order_are_unchanged_by_the_markers(builder): + """AC-6a. The pre-generation payload IS the pre-session payload (AC-2b), so it is + the honest baseline for what the model used to read.""" + marked = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + bare = _make(builder, "gpt-5.5").format_request_payload(CONVERSATION) + assert [i.get("role") for i in _items(marked)] == [i.get("role") for i in _items(bare)] + assert [i.get("type") for i in _items(marked)] == [i.get("type") for i in _items(bare)] + assert [_texts(i) for i in _items(marked)] == [_texts(i) for i in _items(bare)] + + +def test_stripping_the_markers_recovers_the_earlier_payload_byte_for_byte(builder): + """AC-6b. Over the serialized body the only differences are the two cache fields and + the promotion of a string to a one-block list where a marker was placed.""" + marked = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + bare = _make(builder, "gpt-5.5").format_request_payload(CONVERSATION) + # The model field is the one legitimate difference: the two builds are of the same + # conversation for two different models, which is what puts them on two sides of the + # generation gate in the first place. + assert _without_markers(marked) | {"model": None} == bare | {"model": None} + + +# --- AC-7: determinism and the caller's dicts --------------------------------------- + + +@pytest.mark.parametrize("model", SUPPORTING + NON_SUPPORTING) +def test_building_twice_yields_the_same_bytes(builder, model): + """AC-7a.""" + adapter = _make(builder, model) + first = adapter.format_request_payload(CONVERSATION) + second = adapter.format_request_payload(CONVERSATION) + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + + +def test_marking_never_mutates_the_callers_dicts(builder): + """AC-7b. The durable conversation shares these dicts, so a marker stamped in place + would leak into persisted rows — and a second build would then differ from the + first.""" + rows = [ + {"role": "system", "content": "head"}, + {"role": "user", "content": [{"type": "input_text", "text": "shared block"}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": "now 09:00", CACHE_EXEMPT_KEY: True}, + ] + before = json.dumps(rows, sort_keys=True) + _make(builder, "gpt-5.6-terra").format_request_payload(rows) + assert json.dumps(rows, sort_keys=True) == before + + +# --- AC-8: the two re-hosted surfaces ------------------------------------------------ + + +@pytest.mark.parametrize("model", SUPPORTING + NON_SUPPORTING) +def test_azure_and_first_party_build_the_same_body(model): + """AC-8a. Azure overrides no payload builder; the only difference between the two + payloads is which deployment or model name the caller asked for, and here they are + given the same one.""" + first_party = _make(OpenAIAdapter, model).format_request_payload(CONVERSATION) + azure = _make(AzureOpenAIAdapter, model).format_request_payload(CONVERSATION) + assert first_party == azure + + +@pytest.mark.parametrize("adapter_type", OAUTH) +@pytest.mark.parametrize("model", SUPPORTING + NON_SUPPORTING) +def test_the_chatgpt_backend_is_told_nothing_new(adapter_type, model): + """AC-8b. The OAuth twin speaks to ``chatgpt.com/backend-api/codex/responses``, + which neither provider page covers, so it is untouched by this session — including + for the 5.6-shaped names the existing tests already construct on it.""" + payload = _make(adapter_type, model).format_request_payload(CONVERSATION) + assert "prompt_cache_options" not in payload + assert "prompt_cache_breakpoint" not in json.dumps(payload) + outputs = [i for i in payload["input"] if i.get("type") == "function_call_output"] + assert [i["output"] for i in outputs] == ["three rooms free"] + + +# --- AC-9: the flag's new home ------------------------------------------------------- + + +def test_the_exempt_flag_is_importable_from_both_names(): + """AC-9b. It moved to ``base`` once a second adapter family began reading it, and + the Anthropic module re-exports it because that is where callers import it from.""" + assert CACHE_EXEMPT_KEY == ANTHROPIC_CACHE_EXEMPT_KEY == "cache_exempt" + + +# --- the rule that makes the whole thing work ---------------------------------------- + + +def test_a_growing_conversation_keeps_every_marker_it_had(builder): + """The prefix-stability invariant, stated as a test because violating it is the one + failure that looks correct and reads zero. Applying the rule to a conversation and + to that conversation plus a new round must mark the same rows on the shared prefix. + """ + adapter = _make(builder, "gpt-5.6-terra") + step = list(CONVERSATION[:-1]) + volatile = CONVERSATION[-1] + first = adapter.format_request_payload(step + [volatile]) + grown = step + [ + {"role": "assistant", "content": None, + "tool_calls": [{"id": "call_2", "function": {"name": "book", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "call_2", "content": "booked"}, + ] + second = adapter.format_request_payload(grown + [volatile]) + shared = len(_items(first)) - 1 # everything but the volatile row + assert _items(second)[:shared] == _items(first)[:shared] + + +def test_every_durable_markable_row_is_marked(builder): + """Density is free — a breakpoint costs no prompt tokens (measured: the same 10,739 + input tokens with no markers and with six) — and it is what makes the row that was + last on the previous request marked by construction.""" + payload = _make(builder, "gpt-5.6-terra").format_request_payload(CONVERSATION) + markable = [ + i for i, item in enumerate(_items(payload)) + if item.get("type") == "function_call_output" or item.get("role") in {"system", "developer", "user"} + ] + volatile = len(_items(payload)) - 1 + assert [i for i, _ in _markers(payload)] == [i for i in markable if i != volatile] diff --git a/tests/models/test_prompt_cache_breakpoint.py b/tests/models/test_prompt_cache_breakpoint.py index 85008d27..429ee51e 100644 --- a/tests/models/test_prompt_cache_breakpoint.py +++ b/tests/models/test_prompt_cache_breakpoint.py @@ -629,3 +629,72 @@ def test_the_exemption_is_deterministic_and_idempotent(): first = _api().format_request_payload([dict(m) for m in messages]) second = _api().format_request_payload([dict(m) for m in messages]) assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + + +# ── the flag is family-neutral now, and the other family stays exactly as it was ───── + + +def test_the_flag_has_one_value_under_both_names(): + """It moved to ``base`` when a second adapter family began reading it. The Anthropic + name is kept because that is where every existing caller imports it from, and a + dropped re-export would break ``anthropic_oauth`` on import.""" + from marsys.models.adapters import anthropic as anthropic_module + from marsys.models.adapters import base as base_module + + assert anthropic_module.CACHE_EXEMPT_KEY is base_module.CACHE_EXEMPT_KEY + assert base_module.CACHE_EXEMPT_KEY == "cache_exempt" + + +ANTHROPIC_CONVERSATION = [ + {"role": "system", "content": "You are Spren, the founder's employee."}, + {"role": "user", "content": "Book the room."}, + { + "role": "assistant", + "content": "on it", + "tool_calls": [{"id": "call_1", "function": {"name": "search", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "three rooms free"}, + {"role": "user", "content": "now 09:00 | spend 1.20", CACHE_EXEMPT_KEY: True}, +] + + +@pytest.mark.parametrize("model_name", ["claude-opus-5", "gpt-5.6-terra"]) +def test_the_anthropic_family_learns_nothing_about_the_openai_fields(model_name): + """The OpenAI family's explicit breakpoints are a different provider's mechanism + with a different placement rule. Nothing about them may leak into these payloads — + including under a GPT-shaped name, because the model-family gate that scopes them + lives in the other builder and must not be reachable from here at all.""" + for payload in ( + _api(model_name).format_request_payload(ANTHROPIC_CONVERSATION), + _oauth().format_request_payload(ANTHROPIC_CONVERSATION), + BedrockAdapter(model_name=model_name, api_key="tok").format_request_payload( + ANTHROPIC_CONVERSATION + ), + ): + serialized = json.dumps(payload) + assert "prompt_cache_options" not in payload + assert "prompt_cache_breakpoint" not in serialized + assert "input_text" not in serialized + + +@pytest.mark.parametrize("adapter_of", [ + lambda: _api(), + lambda: _oauth(), + lambda: BedrockAdapter(model_name="claude-opus-5", api_key="tok"), +]) +def test_the_anthropic_conversation_marker_is_still_one_and_still_on_the_tail(adapter_of): + """AC-9's substance: one conversation marker, on the last durable row, exactly where + it was. The OAuth leg's static Claude-Code prefix block carries the other marker and + is not part of the conversation.""" + payload = adapter_of().format_request_payload(ANTHROPIC_CONVERSATION) + marked = [ + block for message in payload["messages"] + if isinstance(message.get("content"), list) + for block in message["content"] + if isinstance(block, dict) and "cache_control" in block + ] + assert len(marked) == 1 + assert marked[0]["cache_control"] == EPHEMERAL + # The tool result, which is the last durable row here — the volatile row after it is + # exempt, and a tool row converts into a trailing user message. + assert marked[0].get("type") == "tool_result" diff --git a/tests/models/test_prompt_cache_key.py b/tests/models/test_prompt_cache_key.py deleted file mode 100644 index cef8c59b..00000000 --- a/tests/models/test_prompt_cache_key.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Caller-selected cache routing survives each OpenAI-family payload builder.""" - -import pytest - -from marsys.models.adapters.azure import AsyncAzureOpenAIAdapter, AzureOpenAIAdapter -from marsys.models.adapters.openai import AsyncOpenAIAdapter, OpenAIAdapter -from marsys.models.adapters.openai_oauth import AsyncOpenAIOAuthAdapter, OpenAIOAuthAdapter - - -@pytest.fixture(params=[ - OpenAIAdapter, AsyncOpenAIAdapter, AzureOpenAIAdapter, AsyncAzureOpenAIAdapter, - OpenAIOAuthAdapter, AsyncOpenAIOAuthAdapter, -]) -def adapter(request): - adapter_type = request.param - if issubclass(adapter_type, OpenAIOAuthAdapter): - # Payload construction needs no credential discovery or network client. - result = object.__new__(adapter_type) - result.model_name = "gpt-5" - return result - return adapter_type( - model_name="gpt-5", api_key="not-a-real-key", - base_url="https://example.invalid/openai/v1", max_tokens=1024, - ) - - -def test_caller_key_reaches_payload_and_stays_stable(adapter): - messages = [{"role": "user", "content": "Continue the work."}] - key = "installation:instance" - first = adapter.format_request_payload(messages, prompt_cache_key=key) - second = adapter.format_request_payload(messages, prompt_cache_key=key) - assert first["prompt_cache_key"] == second["prompt_cache_key"] == key - other = adapter.format_request_payload(messages, prompt_cache_key="installation:other") - assert other["prompt_cache_key"] == "installation:other" - - -@pytest.mark.parametrize("kwargs", [{}, {"prompt_cache_key": None}]) -def test_absent_key_is_omitted_without_minting_one(adapter, kwargs): - messages = [{"role": "user", "content": "Continue the work."}] - first = adapter.format_request_payload(messages, **kwargs) - second = adapter.format_request_payload(messages, **kwargs) - assert "prompt_cache_key" not in first - assert "prompt_cache_key" not in second - assert first == second