diff --git a/docs-website/docs/token-counters/tiktokencounter.mdx b/docs-website/docs/token-counters/tiktokencounter.mdx index 87a894d56dc..85b593cfa8e 100644 --- a/docs-website/docs/token-counters/tiktokencounter.mdx +++ b/docs-website/docs/token-counters/tiktokencounter.mdx @@ -58,6 +58,8 @@ The counter loads its encoding on the first call to `count()`. To load it during counter.warm_up() ``` +Literal special-token strings such as `<|endoftext|>` in message or tool content are tokenized as ordinary text. They are counted instead of raising an error. + To include the context consumed by tool schemas, pass the tools to `count()`: ```python diff --git a/haystack/token_counters/tiktoken_counter.py b/haystack/token_counters/tiktoken_counter.py index 57298234074..6eb48ad6bcf 100644 --- a/haystack/token_counters/tiktoken_counter.py +++ b/haystack/token_counters/tiktoken_counter.py @@ -24,6 +24,9 @@ class TiktokenCounter(TokenCounter): than a real count. - **It is OpenAI's encoder.** Other providers tokenize differently, so expect the count to drift on them. + Literal special-token strings (for example `<|endoftext|>`) in message or tool content are encoded as ordinary + text, so they are counted instead of raising an error. + ## Usage Example: ```python from haystack.dataclasses import ChatMessage @@ -74,7 +77,7 @@ def count(self, messages: list[ChatMessage], tools: ToolsType | None = None) -> if not messages and not tools: return 0 self.warm_up() - text_tokens = len(self._encoder.encode(_rendered_conversation(messages) + _rendered_tools(tools))) + text_tokens = len(self._encoder.encode_ordinary(_rendered_conversation(messages) + _rendered_tools(tools))) return text_tokens + _non_text_tokens( messages, tokens_per_image=self.tokens_per_image, tokens_per_file=self.tokens_per_file ) diff --git a/releasenotes/notes/fix-literal-special-tokens-in-tiktoken-counter-9f2c1a7b4e5d8f0a.yaml b/releasenotes/notes/fix-literal-special-tokens-in-tiktoken-counter-9f2c1a7b4e5d8f0a.yaml new file mode 100644 index 00000000000..41120b384c1 --- /dev/null +++ b/releasenotes/notes/fix-literal-special-tokens-in-tiktoken-counter-9f2c1a7b4e5d8f0a.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + ``TiktokenCounter`` no longer raises ``ValueError`` when message or tool content contains literal + special-token strings such as ``<|endoftext|>``. They are now encoded as ordinary text and counted, + consistent with ``DocumentSplitter`` and ``RecursiveDocumentSplitter``. diff --git a/test/token_counters/test_tiktoken_counter.py b/test/token_counters/test_tiktoken_counter.py index bd19f1e5871..a9c6b5240ba 100644 --- a/test/token_counters/test_tiktoken_counter.py +++ b/test/token_counters/test_tiktoken_counter.py @@ -27,7 +27,7 @@ class _FakeEncoder: def __init__(self) -> None: self.encoded: list[str] = [] - def encode(self, text: str) -> list[int]: + def encode_ordinary(self, text: str) -> list[int]: self.encoded.append(text) return list(range(len(text.split()))) @@ -160,3 +160,25 @@ def test_an_image_is_charged_at_the_flat_rate(self): counter = TiktokenCounter(tokens_per_image=85) assert counter.count([ChatMessage.from_user(content_parts=[IMAGE])]) > 85 + + def test_special_token_strings_are_counted_as_literal_text(self): + # Regression test for https://github.com/deepset-ai/haystack/issues/12869: a literal "<|endoftext|>" + # in message content must be counted as ordinary text instead of raising ValueError. + counter = TiktokenCounter() + with_marker = [ChatMessage.from_user("The manual documents <|endoftext|> as a literal marker.")] + without_marker = [ChatMessage.from_user("The manual documents a literal marker.")] + + assert counter.count(with_marker) > counter.count(without_marker) > 0 + + def test_special_token_strings_in_tool_schemas_are_counted_as_literal_text(self): + # Same crash through the tools path: schema text is concatenated into the same encoded string. + + @tool + def lookup(query: Annotated[str, "The <|endoftext|> marker, documented literally"]) -> str: + """Look up the <|endoftext|> marker.""" + return "result" + + counter = TiktokenCounter() + messages = [ChatMessage.from_user("hi")] + + assert counter.count(messages, tools=[lookup]) > counter.count(messages) > 0