fix: count literal special-token strings in TiktokenCounter

haystack/token_counters/tiktoken_counter.py

@@ -74,7 +74,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) ->
         if not messages and not tools:
             return 0
         self.warm_up()
-        text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools)))
+        text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools)))
         return text_tokens + _non_text_tokens(
             messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file
         )

releasenotes/notes/count-literal-special-tokens-e64733d364ddd53e.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed ``TiktokenCounter`` raising a ``ValueError`` when messages or tool schemas contain literal tiktoken
+    special-token strings such as ``<|endoftext|>``. Literal markers are now counted as ordinary text.

test/token_counters/test_tiktoken_counter.py

@@ -27,7 +27,7 @@ class _FakeEncoder:
     def __init__(self) -> None:
         self.encoded: list[str] = []
 
-    def encode(self, text: str) -> list[int]:
+    def encode_ordinary(self, text: str) -> list[int]:
         self.encoded.append(text)
         return list(range(len(text.split())))
 
@@ -116,6 +116,12 @@ def search(query: Annotated[str, "the search query"]) -> str:
     return "result"
 
 
+@tool
+def describe_marker() -> str:
+    """Describe the literal <|endoftext|> marker."""
+    return "marker"
+
+
 class TestTiktokenCounterTools:
     def test_tool_schemas_add_to_the_count(self, fake_encoder):
         # A provider is sent the schemas alongside the messages, so they consume tokens too.
@@ -136,6 +142,16 @@ def test_nothing_to_measure_is_zero(self):
 class TestTiktokenCounterIntegration:
     """Exercises the real encoder, which downloads its vocabulary on first use."""
 
+    def test_counts_literal_special_token_in_message_as_ordinary_text(self):
+        counter = TiktokenCounter()
+
+        assert counter.count([ChatMessage.from_user("The manual documents <|endoftext|> as a literal marker.")]) > 0
+
+    def test_counts_literal_special_token_in_tool_description_as_ordinary_text(self):
+        counter = TiktokenCounter()
+
+        assert counter.count([], tools=[describe_marker]) > 0
+
     def test_counts_grow_with_content(self):
         counter = TiktokenCounter()