diff --git a/app/prompt_cache.py b/app/prompt_cache.py index 7b0a908..6bb983c 100644 --- a/app/prompt_cache.py +++ b/app/prompt_cache.py @@ -69,7 +69,11 @@ def cache_key( "response_format": response_format or None, "seed": seed, "max_tokens": max_tokens, - "stop": stop, + # Normalize stop to a list: "stop": "foo" and "stop": ["foo"] + # produce the same upstream behavior, so they must share a cache + # entry. Without this, a string-vs-list difference fragments the + # cache for semantically identical requests. + "stop": [stop] if isinstance(stop, str) else stop, "tool_choice": tool_choice, "top_p": top_p, "n": n, diff --git a/tests/unit/test_cache_stop_normalization.py b/tests/unit/test_cache_stop_normalization.py new file mode 100644 index 0000000..84ff3f3 --- /dev/null +++ b/tests/unit/test_cache_stop_normalization.py @@ -0,0 +1,40 @@ +"""Regression: prompt cache key must normalize stop to a list. + +OpenAI wire format accepts stop as either a string or a list of strings. +stop="foo" and stop=["foo"] produce identical upstream behavior, so +they must map to the same cache key. Without normalization, the two forms +produce different SHA-256 digests and the cache is fragmented.""" + +from app.prompt_cache import cache_key + + +def _base_kwargs(**overrides): + kw = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hello"}], + "temperature": 0, + "tools": None, + "response_format": None, + "seed": 42, + } + kw.update(overrides) + return kw + + +def test_stop_string_vs_list_same_key(): + k1 = cache_key(**_base_kwargs(stop="END")) + k2 = cache_key(**_base_kwargs(stop=["END"])) + assert k1 == k2 + + +def test_stop_list_order_matters(): + k1 = cache_key(**_base_kwargs(stop=["a", "b"])) + k2 = cache_key(**_base_kwargs(stop=["b", "a"])) + assert k1 != k2 + + +def test_stop_multi_element_list_stable(): + kw = _base_kwargs(stop=["a", "b", "c"]) + k1 = cache_key(**kw) + k2 = cache_key(**kw) + assert k1 == k2