diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py index 0636b3428..1ab7697a2 100644 --- a/python/freetoken/tokenizer/tokenize.py +++ b/python/freetoken/tokenizer/tokenize.py @@ -46,6 +46,19 @@ def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: An _EFFORT_PROBE_MESSAGES = [{"role": "user", "content": "ping"}] +def _map_developer_role( + messages: list[dict[str, Any]], chat_template: str | None +) -> list[dict[str, Any]]: + """OpenAI's ``developer`` role is the current spelling of ``system`` (clients such as pi + send the system prompt that way by default). Most chat templates do not know it and raise + ("Unexpected message role"), so map it to ``system`` unless the template handles it.""" + if "developer" in (chat_template or "") or not any( + m.get("role") == "developer" for m in messages + ): + return messages + return [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages] + + class TokenizeManager: def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None: self.tokenizer = tokenizer @@ -108,6 +121,7 @@ def _render( ) if tools is not None: chat_template_kwargs = {**chat_template_kwargs, "tools": tools} + messages = _map_developer_role(messages, getattr(self.tokenizer, "chat_template", None)) prompt = self.tokenizer.apply_chat_template( messages, tokenize=False, diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.py index 65fa5f2df..2ad4b9199 100644 --- a/tests/tokenizer/test_tokenize.py +++ b/tests/tokenizer/test_tokenize.py @@ -285,3 +285,14 @@ def test_tokenize_survives_an_unhashable_effort(): manager.tokenize([msg]) assert "reasoning_effort" not in tokenizer.chat_template_kwargs + + +def test_developer_role_maps_to_system_unless_the_template_knows_it(): + from freetoken.tokenizer.tokenize import _map_developer_role + + msgs = [{"role": "developer", "content": "be brief"}, {"role": "user", "content": "hi"}] + mapped = _map_developer_role(msgs, "{% if message.role == 'system' %}") + assert [m["role"] for m in mapped] == ["system", "user"] + assert mapped[0]["content"] == "be brief" and msgs[0]["role"] == "developer" # input untouched + assert _map_developer_role(msgs, "{% if message.role == 'developer' %}") is msgs + assert _map_developer_role(msgs[1:], None) == msgs[1:] # nothing to map