From 47a19e334d4739127bf7c5a7c8d396f15aa794df Mon Sep 17 00:00:00 2001 From: ramashishmaurya Date: Tue, 29 Sep 2026 14:42:36 +0530 Subject: [PATCH] Fix ValueError on literal special-token strings in TiktokenCounter (#12869) --- haystack/token_counters/tiktoken_counter.py | 2 +- test/token_counters/test_tiktoken_counter.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/haystack/token_counters/tiktoken_counter.py b/haystack/token_counters/tiktoken_counter.py index 57298234074..f505b89bf99 100644 --- a/haystack/token_counters/tiktoken_counter.py +++ b/haystack/token_counters/tiktoken_counter.py @@ -74,7 +74,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) -> if not messages and not tools: return 0 self.warm_up() - text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools))) + text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools))) return text_tokens + _non_text_tokens( messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file ) diff --git a/test/token_counters/test_tiktoken_counter.py b/test/token_counters/test_tiktoken_counter.py index bd19f1e5871..127fc57f45b 100644 --- a/test/token_counters/test_tiktoken_counter.py +++ b/test/token_counters/test_tiktoken_counter.py @@ -136,6 +136,11 @@ def test_nothing_to_measure_is_zero(self): class TestTiktokenCounterIntegration: """Exercises the real encoder, which downloads its vocabulary on first use.""" + def test_counts_literal_special_tokens(self): + counter = TiktokenCounter() + count = counter.count([ChatMessage.from_user("The manual documents <|endoftext|> as a literal marker.")]) + assert count > 0 + def test_counts_grow_with_content(self): counter = TiktokenCounter()