fix: count literal special-token strings in TiktokenCounter as ordinary text

docs-website/docs/token-counters/tiktokencounter.mdx

@@ -77,3 +77,7 @@ counter = TiktokenCounter(
 ```
 
 The counter includes non-text content attached directly to a message as well as content nested inside tool results.
+
+## Special-token strings
+
+Strings that look like special tokens, such as `<|endoftext|>`, are counted as ordinary text. They are measured like any other source text, so a message or tool schema that contains one is not rejected.

haystack/token_counters/tiktoken_counter.py

@@ -24,6 +24,9 @@ class TiktokenCounter(TokenCounter):
       than a real count.
     - **It is OpenAI's encoder.** Other providers tokenize differently, so expect the count to drift on them.
 
+    Strings that look like special tokens, such as `<|endoftext|>`, are counted as ordinary text instead of being
+    rejected.
+
     ## Usage Example:
     ```python
     from haystack.dataclasses import ChatMessage
@@ -74,7 +77,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) ->
         if not messages and not tools:
             return 0
         self.warm_up()
-        text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools)))
+        text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools)))
         return text_tokens + _non_text_tokens(
             messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file
         )

releasenotes/notes/fix-tiktoken-counter-literal-special-tokens-cfcc252a1fa82f84.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fix ``TiktokenCounter`` failing on messages or tool schemas containing strings such as
+    ``<|endoftext|>``. These strings are now counted as ordinary text instead of raising a
+    ``ValueError``.

test/token_counters/test_tiktoken_counter.py

@@ -27,7 +27,7 @@ class _FakeEncoder:
     def __init__(self) -> None:
         self.encoded: list[str] = []
 
-    def encode(self, text: str) -> list[int]:
+    def encode_ordinary(self, text: str) -> list[int]:
         self.encoded.append(text)
         return list(range(len(text.split())))
 
@@ -160,3 +160,14 @@ def test_an_image_is_charged_at_the_flat_rate(self):
         counter = TiktokenCounter(tokens_per_image=85)
 
         assert counter.count([ChatMessage.from_user(content_parts=[IMAGE])]) > 85
+
+    @pytest.mark.parametrize("encoding", ["o200k_base", "cl100k_base"])
+    def test_special_token_strings_are_counted_as_ordinary_text(self, encoding):
+        # `<|endoftext|>` is a tiktoken special token. In source text it is an ordinary string, so the counter has
+        # to measure it rather than reject the message that contains it.
+        counter = TiktokenCounter(encoding=encoding)
+
+        with_marker = counter.count([ChatMessage.from_user("the manual marks <|endoftext|> as literal")])
+        without_marker = counter.count([ChatMessage.from_user("the manual marks as literal")])
+
+        assert with_marker > without_marker