fix: count literal special-token strings as ordinary text in TiktokenCounter

docs-website/docs/token-counters/tiktokencounter.mdx

@@ -58,6 +58,8 @@ The counter loads its encoding on the first call to `count()`. To load it during
 counter.warm_up()
 ```
 
+Literal special-token strings such as `<|endoftext|>` in message or tool content are tokenized as ordinary text. They are counted instead of raising an error.
+
 To include the context consumed by tool schemas, pass the tools to `count()`:
 
 ```python

haystack/token_counters/tiktoken_counter.py

@@ -24,6 +24,9 @@ class TiktokenCounter(TokenCounter):
       than a real count.
     - **It is OpenAI's encoder.** Other providers tokenize differently, so expect the count to drift on them.
 
+    Literal special-token strings (for example `<|endoftext|>`) in message or tool content are encoded as ordinary
+    text, so they are counted instead of raising an error.
+
     ## Usage Example:
     ```python
     from haystack.dataclasses import ChatMessage
@@ -74,7 +77,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) ->
         if not messages and not tools:
             return 0
         self.warm_up()
-        text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools)))
+        text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools)))
         return text_tokens + _non_text_tokens(
             messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file
         )

releasenotes/notes/fix-literal-special-tokens-in-tiktoken-counter-9f2c1a7b4e5d8f0a.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    ``TiktokenCounter`` no longer raises ``ValueError`` when message or tool content contains literal
+    special-token strings such as ``<|endoftext|>``. They are now encoded as ordinary text and counted,
+    consistent with ``DocumentSplitter`` and ``RecursiveDocumentSplitter``.

test/token_counters/test_tiktoken_counter.py

@@ -27,7 +27,7 @@ class _FakeEncoder:
     def __init__(self) -> None:
         self.encoded: list[str] = []
 
-    def encode(self, text: str) -> list[int]:
+    def encode_ordinary(self, text: str) -> list[int]:
         self.encoded.append(text)
         return list(range(len(text.split())))
 
@@ -160,3 +160,25 @@ def test_an_image_is_charged_at_the_flat_rate(self):
         counter = TiktokenCounter(tokens_per_image=85)
 
         assert counter.count([ChatMessage.from_user(content_parts=[IMAGE])]) > 85
+
+    def test_special_token_strings_are_counted_as_literal_text(self):
+        # Regression test for https://github.com/deepset-ai/haystack/issues/12869: a literal "<|endoftext|>"
+        # in message content must be counted as ordinary text instead of raising ValueError.
+        counter = TiktokenCounter()
+        with_marker = [ChatMessage.from_user("The manual documents <|endoftext|> as a literal marker.")]
+        without_marker = [ChatMessage.from_user("The manual documents a literal marker.")]
+
+        assert counter.count(with_marker) > counter.count(without_marker) > 0
+
+    def test_special_token_strings_in_tool_schemas_are_counted_as_literal_text(self):
+        # Same crash through the tools path: schema text is concatenated into the same encoded string.
+
+        @tool
+        def lookup(query: Annotated[str, "The <|endoftext|> marker, documented literally"]) -> str:
+            """Look up the <|endoftext|> marker."""
+            return "result"
+
+        counter = TiktokenCounter()
+        messages = [ChatMessage.from_user("hi")]
+
+        assert counter.count(messages, tools=[lookup]) > counter.count(messages) > 0