From 965e178071d92297253c66a5cb00acfe65e761d7 Mon Sep 17 00:00:00 2001 From: "Gabriel A. Devenyi" Date: Fri, 4 Sep 2026 20:24:22 -0400 Subject: [PATCH] fix(tokenizer): accept the OpenAI developer role (map to system for templates without it) Clients such as pi send the system prompt with role "developer" by default (OpenAI's current spelling of "system"). Most chat templates do not know the role and raise "Unexpected message role", which the server returns as a 400. Map developer to system before rendering unless the template spells the role itself (gpt-oss does). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0173pf9k9fSVtwbm3f898HDt --- python/freetoken/tokenizer/tokenize.py | 14 ++++++++++++++ tests/tokenizer/test_tokenize.py | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py index 0636b3428..1ab7697a2 100644 --- a/python/freetoken/tokenizer/tokenize.py +++ b/python/freetoken/tokenizer/tokenize.py @@ -46,6 +46,19 @@ def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: An _EFFORT_PROBE_MESSAGES = [{"role": "user", "content": "ping"}] +def _map_developer_role( + messages: list[dict[str, Any]], chat_template: str | None +) -> list[dict[str, Any]]: + """OpenAI's ``developer`` role is the current spelling of ``system`` (clients such as pi + send the system prompt that way by default). Most chat templates do not know it and raise + ("Unexpected message role"), so map it to ``system`` unless the template handles it.""" + if "developer" in (chat_template or "") or not any( + m.get("role") == "developer" for m in messages + ): + return messages + return [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages] + + class TokenizeManager: def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None: self.tokenizer = tokenizer @@ -108,6 +121,7 @@ def _render( ) if tools is not None: chat_template_kwargs = {**chat_template_kwargs, "tools": tools} + messages = _map_developer_role(messages, getattr(self.tokenizer, "chat_template", None)) prompt = self.tokenizer.apply_chat_template( messages, tokenize=False, diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.py index 65fa5f2df..2ad4b9199 100644 --- a/tests/tokenizer/test_tokenize.py +++ b/tests/tokenizer/test_tokenize.py @@ -285,3 +285,14 @@ def test_tokenize_survives_an_unhashable_effort(): manager.tokenize([msg]) assert "reasoning_effort" not in tokenizer.chat_template_kwargs + + +def test_developer_role_maps_to_system_unless_the_template_knows_it(): + from freetoken.tokenizer.tokenize import _map_developer_role + + msgs = [{"role": "developer", "content": "be brief"}, {"role": "user", "content": "hi"}] + mapped = _map_developer_role(msgs, "{% if message.role == 'system' %}") + assert [m["role"] for m in mapped] == ["system", "user"] + assert mapped[0]["content"] == "be brief" and msgs[0]["role"] == "developer" # input untouched + assert _map_developer_role(msgs, "{% if message.role == 'developer' %}") is msgs + assert _map_developer_role(msgs[1:], None) == msgs[1:] # nothing to map