fix(tokenizer): accept the OpenAI developer role (map to system for templates without it)

python/freetoken/tokenizer/tokenize.py

@@ -46,6 +46,19 @@ def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: An
 _EFFORT_PROBE_MESSAGES = [{"role": "user", "content": "ping"}]
 
 
+def _map_developer_role(
+    messages: list[dict[str, Any]], chat_template: str | None
+) -> list[dict[str, Any]]:
+    """OpenAI's ``developer`` role is the current spelling of ``system`` (clients such as pi
+    send the system prompt that way by default). Most chat templates do not know it and raise
+    ("Unexpected message role"), so map it to ``system`` unless the template handles it."""
+    if "developer" in (chat_template or "") or not any(
+        m.get("role") == "developer" for m in messages
+    ):
+        return messages
+    return [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages]
+
+
 class TokenizeManager:
     def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None:
         self.tokenizer = tokenizer
@@ -108,6 +121,7 @@ def _render(
             )
         if tools is not None:
             chat_template_kwargs = {**chat_template_kwargs, "tools": tools}
+        messages = _map_developer_role(messages, getattr(self.tokenizer, "chat_template", None))
         prompt = self.tokenizer.apply_chat_template(
             messages,
             tokenize=False,

tests/tokenizer/test_tokenize.py

@@ -285,3 +285,14 @@ def test_tokenize_survives_an_unhashable_effort():
     manager.tokenize([msg])
 
     assert "reasoning_effort" not in tokenizer.chat_template_kwargs
+
+
+def test_developer_role_maps_to_system_unless_the_template_knows_it():
+    from freetoken.tokenizer.tokenize import _map_developer_role
+
+    msgs = [{"role": "developer", "content": "be brief"}, {"role": "user", "content": "hi"}]
+    mapped = _map_developer_role(msgs, "{% if message.role == 'system' %}")
+    assert [m["role"] for m in mapped] == ["system", "user"]
+    assert mapped[0]["content"] == "be brief" and msgs[0]["role"] == "developer"  # input untouched
+    assert _map_developer_role(msgs, "{% if message.role == 'developer' %}") is msgs
+    assert _map_developer_role(msgs[1:], None) == msgs[1:]  # nothing to map