From 44959f04313ff028379cf970f11df5cfe06e0ced Mon Sep 17 00:00:00 2001
From: hahahahahayesyeseys <1145018948@qq.com>
Date: Wed, 23 Sep 2026 11:31:21 +0000
Subject: [PATCH] fix: count literal special-token strings in TiktokenCounter

---
 haystack/token_counters/tiktoken_counter.py    |  2 +-
 ...iteral-special-tokens-e64733d364ddd53e.yaml |  5 +++++
 test/token_counters/test_tiktoken_counter.py   | 18 +++++++++++++++++-
 3 files changed, 23 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/count-literal-special-tokens-e64733d364ddd53e.yaml

diff --git a/haystack/token_counters/tiktoken_counter.py b/haystack/token_counters/tiktoken_counter.py
index 57298234074..f505b89bf99 100644
--- a/haystack/token_counters/tiktoken_counter.py
+++ b/haystack/token_counters/tiktoken_counter.py
@@ -74,7 +74,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) ->
         if not messages and not tools:
             return 0
         self.warm_up()
-        text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools)))
+        text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools)))
         return text_tokens + _non_text_tokens(
             messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file
         )
diff --git a/releasenotes/notes/count-literal-special-tokens-e64733d364ddd53e.yaml b/releasenotes/notes/count-literal-special-tokens-e64733d364ddd53e.yaml
new file mode 100644
index 00000000000..c39c243f8d9
--- /dev/null
+++ b/releasenotes/notes/count-literal-special-tokens-e64733d364ddd53e.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed ``TiktokenCounter`` raising a ``ValueError`` when messages or tool schemas contain literal tiktoken
+    special-token strings such as ``<|endoftext|>``. Literal markers are now counted as ordinary text.
diff --git a/test/token_counters/test_tiktoken_counter.py b/test/token_counters/test_tiktoken_counter.py
index bd19f1e5871..581b8afefb2 100644
--- a/test/token_counters/test_tiktoken_counter.py
+++ b/test/token_counters/test_tiktoken_counter.py
@@ -27,7 +27,7 @@ class _FakeEncoder:
     def __init__(self) -> None:
         self.encoded: list[str] = []
 
-    def encode(self, text: str) -> list[int]:
+    def encode_ordinary(self, text: str) -> list[int]:
         self.encoded.append(text)
         return list(range(len(text.split())))
 
@@ -116,6 +116,12 @@ def search(query: Annotated[str, "the search query"]) -> str:
     return "result"
 
 
+@tool
+def describe_marker() -> str:
+    """Describe the literal <|endoftext|> marker."""
+    return "marker"
+
+
 class TestTiktokenCounterTools:
     def test_tool_schemas_add_to_the_count(self, fake_encoder):
         # A provider is sent the schemas alongside the messages, so they consume tokens too.
@@ -136,6 +142,16 @@ def test_nothing_to_measure_is_zero(self):
 class TestTiktokenCounterIntegration:
     """Exercises the real encoder, which downloads its vocabulary on first use."""
 
+    def test_counts_literal_special_token_in_message_as_ordinary_text(self):
+        counter = TiktokenCounter()
+
+        assert counter.count([ChatMessage.from_user("The manual documents <|endoftext|> as a literal marker.")]) > 0
+
+    def test_counts_literal_special_token_in_tool_description_as_ordinary_text(self):
+        counter = TiktokenCounter()
+
+        assert counter.count([], tools=[describe_marker]) > 0
+
     def test_counts_grow_with_content(self):
         counter = TiktokenCounter()