From d8140279099b5550ff5fa8ffb213cd750d4bfeac Mon Sep 17 00:00:00 2001 From: "Wen.Vale" <2336420857@qq.com> Date: Wed, 23 Sep 2026 19:17:16 +0800 Subject: [PATCH] fix: count literal special-token strings in TiktokenCounter as ordinary text TiktokenCounter.count() encoded the rendered conversation and tool schemas with Encoding.encode(), which defaults to disallowed_special="all" and so raised ValueError on any source text containing a string such as <|endoftext|>. Because allowed_special also defaults to set(), the call could never emit a special token: the strict check could only turn ordinary text into a crash. Encode with encode_ordinary() instead, so such strings are measured as ordinary text. This matches the fix already applied to DocumentSplitter and RecursiveDocumentSplitter in #12853. Refs #12869 Co-Authored-By: Claude Code --- .../docs/token-counters/tiktokencounter.mdx | 4 ++++ haystack/token_counters/tiktoken_counter.py | 5 ++++- ...ter-literal-special-tokens-cfcc252a1fa82f84.yaml | 6 ++++++ test/token_counters/test_tiktoken_counter.py | 13 ++++++++++++- 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/fix-tiktoken-counter-literal-special-tokens-cfcc252a1fa82f84.yaml diff --git a/docs-website/docs/token-counters/tiktokencounter.mdx b/docs-website/docs/token-counters/tiktokencounter.mdx index 87a894d56d..9c874ddf30 100644 --- a/docs-website/docs/token-counters/tiktokencounter.mdx +++ b/docs-website/docs/token-counters/tiktokencounter.mdx @@ -77,3 +77,7 @@ counter = TiktokenCounter( ``` The counter includes non-text content attached directly to a message as well as content nested inside tool results. + +## Special-token strings + +Strings that look like special tokens, such as `<|endoftext|>`, are counted as ordinary text. They are measured like any other source text, so a message or tool schema that contains one is not rejected. diff --git a/haystack/token_counters/tiktoken_counter.py b/haystack/token_counters/tiktoken_counter.py index 5729823407..d7d9875d47 100644 --- a/haystack/token_counters/tiktoken_counter.py +++ b/haystack/token_counters/tiktoken_counter.py @@ -24,6 +24,9 @@ class TiktokenCounter(TokenCounter): than a real count. - **It is OpenAI's encoder.** Other providers tokenize differently, so expect the count to drift on them. + Strings that look like special tokens, such as `<|endoftext|>`, are counted as ordinary text instead of being + rejected. + ## Usage Example: ```python from haystack.dataclasses import ChatMessage @@ -74,7 +77,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) -> if not messages and not tools: return 0 self.warm_up() - text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools))) + text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools))) return text_tokens + _non_text_tokens( messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file ) diff --git a/releasenotes/notes/fix-tiktoken-counter-literal-special-tokens-cfcc252a1fa82f84.yaml b/releasenotes/notes/fix-tiktoken-counter-literal-special-tokens-cfcc252a1fa82f84.yaml new file mode 100644 index 0000000000..0fbbdf09fb --- /dev/null +++ b/releasenotes/notes/fix-tiktoken-counter-literal-special-tokens-cfcc252a1fa82f84.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fix ``TiktokenCounter`` failing on messages or tool schemas containing strings such as + ``<|endoftext|>``. These strings are now counted as ordinary text instead of raising a + ``ValueError``. diff --git a/test/token_counters/test_tiktoken_counter.py b/test/token_counters/test_tiktoken_counter.py index bd19f1e587..72da72820e 100644 --- a/test/token_counters/test_tiktoken_counter.py +++ b/test/token_counters/test_tiktoken_counter.py @@ -27,7 +27,7 @@ class _FakeEncoder: def __init__(self) -> None: self.encoded: list[str] = [] - def encode(self, text: str) -> list[int]: + def encode_ordinary(self, text: str) -> list[int]: self.encoded.append(text) return list(range(len(text.split()))) @@ -160,3 +160,14 @@ def test_an_image_is_charged_at_the_flat_rate(self): counter = TiktokenCounter(tokens_per_image=85) assert counter.count([ChatMessage.from_user(content_parts=[IMAGE])]) > 85 + + @pytest.mark.parametrize("encoding", ["o200k_base", "cl100k_base"]) + def test_special_token_strings_are_counted_as_ordinary_text(self, encoding): + # `<|endoftext|>` is a tiktoken special token. In source text it is an ordinary string, so the counter has + # to measure it rather than reject the message that contains it. + counter = TiktokenCounter(encoding=encoding) + + with_marker = counter.count([ChatMessage.from_user("the manual marks <|endoftext|> as literal")]) + without_marker = counter.count([ChatMessage.from_user("the manual marks as literal")]) + + assert with_marker > without_marker