From 01fbc6cf295794354b09655358460a97d1c579bf Mon Sep 17 00:00:00 2001 From: Omar Goubail Date: Tue, 23 Jun 2026 15:45:43 +0300 Subject: [PATCH] feat(openai): pass through code_interpreter tool in Responses API --- guides/openai.md | 38 +++ lib/req_llm/providers/openai/responses_api.ex | 104 ++++++-- .../openai/responses_api_unit_test.exs | 252 ++++++++++++++++++ 3 files changed, 370 insertions(+), 24 deletions(-) diff --git a/guides/openai.md b/guides/openai.md index fb715d37f..3751cbe31 100644 --- a/guides/openai.md +++ b/guides/openai.md @@ -303,6 +303,44 @@ response.usage.cost Responses API server-side tools may also appear in `response.message.tool_calls` as builtin records (for example `web_search_call` or `file_search_call`). They are preserved for observability, but the provider already executed them: do not replay them as local tool calls. `ReqLLM.Response.classify/1` and `ReqLLM.StreamResponse.classify/1` treat builtin-only responses as final answers. +### Code Interpreter (Responses API) + +Models using the Responses API support the Code Interpreter tool, which runs Python code in a sandboxed container. Pass the tool as a map and ReqLLM will forward it unchanged to OpenAI: + +```elixir +{:ok, response} = ReqLLM.generate_text( + "openai:gpt-5-mini", + "What is the factorial of 12804/53 + 300? Solve with Python.", + tools: [%{ + "type" => "code_interpreter", + "container" => %{"type" => "auto", "memory_limit" => "4g"} + }] +) + +# Access the raw code interpreter output items +response.provider_meta["code_interpreter"]["items"] +#=> [ +#=> %{ +#=> "type" => "code_interpreter_call", +#=> "code" => "from fractions import Fraction...", +#=> "status" => "completed", +#=> ... +#=> } +#=> ] + +# Access code interpreter usage +response.usage.tool_usage.code_interpreter +#=> %{count: 1, unit: :call} +``` + +The container value may also be an existing container ID string: + +```elixir +tools: [%{"type" => "code_interpreter", "container" => "cntr_abc123"}] +``` + +Code Interpreter is a server-side builtin: the provider executes the code and returns the result items. Do not replay them as local tool calls. `ReqLLM.Response.classify/1` treats these responses as final answers. + ### Image Generation Image generation costs are tracked separately: diff --git a/lib/req_llm/providers/openai/responses_api.ex b/lib/req_llm/providers/openai/responses_api.ex index 771f8753b..ef24624ac 100644 --- a/lib/req_llm/providers/openai/responses_api.ex +++ b/lib/req_llm/providers/openai/responses_api.ex @@ -30,6 +30,17 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do - Token limits use `max_output_tokens` instead of `max_tokens` - Tool choice format: `{type: "function", name: "tool_name"}` - Reasoning effort: `{effort: "high"}` format + - **Code Interpreter**: tool maps such as `%{"type" => "code_interpreter", "container" => ...}` + are passed through unchanged to OpenAI. Both object containers + (`%{"type" => "auto", ...}`) and string container IDs are supported. + This is Responses-API-only; Chat Completions does not support the + `code_interpreter` tool type. + + ## Code Interpreter + + Raw `code_interpreter_*` output items from the response are collected in + `response.provider_meta["code_interpreter"]["items"]` and are excluded from + normal text and function tool-call extraction. ## Decoding @@ -59,13 +70,14 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do require Logger require ReqLLM.Debug, as: Debug - @builtin_tool_types ~w(web_search web_search_preview file_search mcp x_search) + @builtin_tool_types ~w(web_search web_search_preview file_search mcp x_search code_interpreter) @tool_usage_type_atoms %{ "web_search" => :web_search, "web_search_preview" => :web_search_preview, "file_search" => :file_search, "mcp" => :mcp, - "x_search" => :x_search + "x_search" => :x_search, + "code_interpreter" => :code_interpreter } @tool_call_atom_keys %{ "web_search_call" => :web_search_call, @@ -266,6 +278,8 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do meta = maybe_put_reasoning_details(meta, extract_reasoning_details_from_segments(response_output)) + meta = merge_code_interpreter_meta(meta, response_output) + meta = merge_response_provider_meta(meta, data["response"] || %{}) [ReqLLM.StreamChunk.meta(meta)] @@ -306,6 +320,20 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do defp merge_response_provider_meta(meta, _), do: meta + defp merge_code_interpreter_meta(meta, response_output) when is_list(response_output) do + items = extract_code_interpreter_items(response_output) + + if items == [] do + meta + else + provider_meta = Map.get(meta, :provider_meta, %{}) + updated = put_code_interpreter_meta(provider_meta, items) + Map.put(meta, :provider_meta, updated) + end + end + + defp merge_code_interpreter_meta(meta, _), do: meta + defp drop_blanks(map) do map |> Enum.reject(fn {_key, value} -> value in [nil, ""] end) @@ -1142,19 +1170,24 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do end type when is_binary(type) -> - if Map.has_key?(@tool_call_atom_keys, type) do - [ - ReqLLM.StreamChunk.meta(%{ - builtin_tool_started: %{ - id: item["id"] || item[:id] || item["call_id"] || item[:call_id], - name: type, - index: stream_output_index(data), - started_at_unix_nano: System.system_time(:nanosecond) - } - }) - ] - else - [] + cond do + code_interpreter_item?(item) -> + [ReqLLM.StreamChunk.meta(%{code_interpreter_item: item})] + + Map.has_key?(@tool_call_atom_keys, type) -> + [ + ReqLLM.StreamChunk.meta(%{ + builtin_tool_started: %{ + id: item["id"] || item[:id] || item["call_id"] || item[:call_id], + name: type, + index: stream_output_index(data), + started_at_unix_nano: System.system_time(:nanosecond) + } + }) + ] + + true -> + [] end _ -> @@ -1175,19 +1208,22 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do defp handle_output_item_done(_, _), do: [] defp handle_output_item_done_item(item, data, state) do - case item["type"] || item[:type] do - "function_call" -> + type = item["type"] || item[:type] + + cond do + type == "function_call" -> handle_function_call_item_done(item, data, state) - "message" -> + type == "message" -> handle_message_item_done(item, data, state) - type when is_binary(type) -> - if Map.has_key?(@tool_call_atom_keys, type), - do: handle_builtin_call_item_done(item, data, state, type), - else: [] + code_interpreter_item?(item) -> + [ReqLLM.StreamChunk.meta(%{code_interpreter_item: item})] - _ -> + is_binary(type) and Map.has_key?(@tool_call_atom_keys, type) -> + handle_builtin_call_item_done(item, data, state, type) + + true -> [] end end @@ -1645,6 +1681,7 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do thinking = aggregate_reasoning_segments(output_segments) tool_calls = extract_tool_calls_from_segments(output_segments) reasoning_details = extract_reasoning_details_from_segments(output_segments) + code_interpreter_items = extract_code_interpreter_items(output_segments) base_usage = %{ input_tokens: get_in(body, ["usage", "input_tokens"]) || 0, @@ -1682,7 +1719,10 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do |> Map.drop(["id", "model", "output_text", "output", "usage"]) |> Map.put("api_type", "responses") - provider_meta = Map.merge(base_provider_meta, object_meta) + provider_meta = + base_provider_meta + |> Map.merge(object_meta) + |> put_code_interpreter_meta(code_interpreter_items) response = %ReqLLM.Response{ id: body["id"] || "unknown", @@ -2026,6 +2066,22 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do defp extract_summary_text(_), do: nil + defp extract_code_interpreter_items(segments) when is_list(segments) do + Enum.filter(segments, &code_interpreter_item?/1) + end + + defp extract_code_interpreter_items(_), do: [] + + defp code_interpreter_item?(%{"type" => "code_interpreter" <> _}), do: true + defp code_interpreter_item?(%{type: "code_interpreter" <> _}), do: true + defp code_interpreter_item?(_), do: false + + defp put_code_interpreter_meta(provider_meta, []), do: provider_meta + + defp put_code_interpreter_meta(provider_meta, items) when is_list(items) do + Map.put(provider_meta, "code_interpreter", %{"items" => items}) + end + defp normalize_arguments_json(nil), do: "{}" defp normalize_arguments_json(""), do: "{}" diff --git a/test/provider/openai/responses_api_unit_test.exs b/test/provider/openai/responses_api_unit_test.exs index 2626e61b3..92a308e54 100644 --- a/test/provider/openai/responses_api_unit_test.exs +++ b/test/provider/openai/responses_api_unit_test.exs @@ -130,6 +130,50 @@ defmodule Provider.OpenAI.ResponsesAPIUnitTest do assert Enum.sort(encoded_tool["parameters"]["required"]) == ["location", "units"] end + test "passes through code_interpreter tool maps unchanged" do + tool = %{ + "type" => "code_interpreter", + "container" => %{"type" => "auto", "memory_limit" => "4g"} + } + + request = build_request(tools: [tool]) + + encoded = ResponsesAPI.encode_body(request) + body = ReqLLM.Test.Helpers.json_body(encoded) + + assert [encoded_tool] = body["tools"] + assert encoded_tool["type"] == "code_interpreter" + assert encoded_tool["container"] == %{"type" => "auto", "memory_limit" => "4g"} + refute Map.has_key?(encoded_tool, "strict") + refute Map.has_key?(encoded_tool, "parameters") + end + + test "passes through atom-keyed code_interpreter tool maps" do + tool = %{type: :code_interpreter, container: %{type: :auto, memory_limit: "4g"}} + + request = build_request(tools: [tool]) + + encoded = ResponsesAPI.encode_body(request) + body = ReqLLM.Test.Helpers.json_body(encoded) + + assert [encoded_tool] = body["tools"] + assert encoded_tool["type"] == "code_interpreter" + assert encoded_tool["container"] == %{"type" => "auto", "memory_limit" => "4g"} + end + + test "passes through string container IDs for code_interpreter" do + tool = %{"type" => "code_interpreter", "container" => "cntr_abc123"} + + request = build_request(tools: [tool]) + + encoded = ResponsesAPI.encode_body(request) + body = ReqLLM.Test.Helpers.json_body(encoded) + + assert [encoded_tool] = body["tools"] + assert encoded_tool["type"] == "code_interpreter" + assert encoded_tool["container"] == "cntr_abc123" + end + test "does not emit unverified-model warnings when the request uses the id field" do warning = ExUnit.CaptureIO.capture_io(:stderr, fn -> @@ -1067,6 +1111,141 @@ defmodule Provider.OpenAI.ResponsesAPIUnitTest do assert resp.body.usage.total_tokens == 0 end + test "collects code_interpreter output items in provider_meta" do + response_body = %{ + "id" => "resp_123", + "model" => "gpt-5", + "output" => [ + %{ + "type" => "message", + "content" => [%{"type" => "output_text", "text" => "The result is 4."}] + }, + %{ + "type" => "code_interpreter_call", + "id" => "ci_xxx", + "code" => "print(2+2)", + "status" => "completed" + }, + %{ + "type" => "code_interpreter_logs", + "call_id" => "ci_xxx", + "logs" => "4\n" + }, + %{ + "type" => "code_interpreter_interpretation", + "call_id" => "ci_xxx", + "text" => "The result is 4." + } + ], + "usage" => %{"input_tokens" => 5, "output_tokens" => 10} + } + + {_req, resp} = ResponsesAPI.decode_response(build_response(200, response_body)) + + assert [part] = resp.body.message.content + assert part.type == :text + assert part.text == "The result is 4." + assert resp.body.message.tool_calls == nil + + assert [call, logs, interpretation] = + get_in(resp.body.provider_meta, ["code_interpreter", "items"]) + + assert call["type"] == "code_interpreter_call" + assert call["id"] == "ci_xxx" + assert logs["type"] == "code_interpreter_logs" + assert logs["logs"] == "4\n" + assert interpretation["type"] == "code_interpreter_interpretation" + assert interpretation["text"] == "The result is 4." + end + + test "preserves code_interpreter_call outputs in provider_meta" do + response_body = %{ + "id" => "resp_123", + "model" => "gpt-5", + "output" => [ + %{ + "type" => "code_interpreter_call", + "id" => "ci_xxx", + "code" => "print(2+2)", + "status" => "completed", + "outputs" => [ + %{"type" => "code_interpreter_logs", "logs" => "4\n"}, + %{"type" => "code_interpreter_interpretation", "text" => "Four."} + ] + } + ], + "usage" => %{"input_tokens" => 5, "output_tokens" => 10} + } + + {_req, resp} = ResponsesAPI.decode_response(build_response(200, response_body)) + + assert [item] = get_in(resp.body.provider_meta, ["code_interpreter", "items"]) + assert item["type"] == "code_interpreter_call" + assert [logs, interpretation] = item["outputs"] + assert logs["type"] == "code_interpreter_logs" + assert logs["logs"] == "4\n" + assert interpretation["type"] == "code_interpreter_interpretation" + assert interpretation["text"] == "Four." + end + + test "counts code_interpreter usage in tool_usage" do + response_body = %{ + "id" => "resp_123", + "model" => "gpt-5", + "output" => [ + %{ + "type" => "code_interpreter_call", + "id" => "ci_xxx", + "code" => "print(2+2)", + "status" => "completed" + } + ], + "usage" => %{"input_tokens" => 5, "output_tokens" => 10} + } + + {_req, resp} = ResponsesAPI.decode_response(build_response(200, response_body)) + + assert resp.body.usage.tool_usage == %{code_interpreter: %{count: 1, unit: :call}} + end + + test "keeps text and function calls separate from code_interpreter items" do + response_body = %{ + "id" => "resp_123", + "model" => "gpt-5", + "output" => [ + %{ + "type" => "message", + "content" => [%{"type" => "output_text", "text" => "The result is 4."}] + }, + %{ + "type" => "function_call", + "call_id" => "call_123", + "name" => "get_weather", + "arguments" => ~s({"location":"SF"}) + }, + %{ + "type" => "code_interpreter_call", + "id" => "ci_xxx", + "code" => "print(2+2)", + "status" => "completed" + } + ], + "usage" => %{"input_tokens" => 5, "output_tokens" => 10} + } + + {_req, resp} = ResponsesAPI.decode_response(build_response(200, response_body)) + + assert [part] = resp.body.message.content + assert part.type == :text + assert part.text == "The result is 4." + + assert [%ReqLLM.ToolCall{function: %{name: "get_weather"}}] = resp.body.message.tool_calls + + assert [call] = get_in(resp.body.provider_meta, ["code_interpreter", "items"]) + assert call["type"] == "code_interpreter_call" + assert call["id"] == "ci_xxx" + end + test "appends message to request context" do msg = %ReqLLM.Message{ role: :user, @@ -1474,6 +1653,79 @@ defmodule Provider.OpenAI.ResponsesAPIUnitTest do assert is_integer(chunk.metadata.done_at_unix_nano) end + test "emits code_interpreter items from output_item.added", %{model: model} do + event = %{ + data: %{ + "event" => "response.output_item.added", + "output_index" => 0, + "item" => %{ + "id" => "ci_xxx", + "type" => "code_interpreter_call", + "code" => "print(2+2)", + "status" => "in_progress" + } + } + } + + assert [%ReqLLM.StreamChunk{type: :meta, metadata: metadata}] = + ResponsesAPI.decode_stream_event(event, model) + + assert metadata.code_interpreter_item["type"] == "code_interpreter_call" + assert metadata.code_interpreter_item["id"] == "ci_xxx" + end + + test "emits code_interpreter items from output_item.done", %{model: model} do + event = %{ + data: %{ + "event" => "response.output_item.done", + "output_index" => 0, + "item" => %{ + "type" => "code_interpreter_logs", + "call_id" => "ci_xxx", + "logs" => "4\n" + } + } + } + + assert [%ReqLLM.StreamChunk{type: :meta, metadata: metadata}] = + ResponsesAPI.decode_stream_event(event, model) + + assert metadata.code_interpreter_item["type"] == "code_interpreter_logs" + assert metadata.code_interpreter_item["logs"] == "4\n" + end + + test "collects code_interpreter items in response.completed provider_meta", %{model: model} do + completed = %{ + data: %{ + "event" => "response.completed", + "response" => %{ + "id" => "resp_123", + "output" => [ + %{ + "type" => "code_interpreter_call", + "id" => "ci_xxx", + "code" => "print(2+2)", + "status" => "completed" + }, + %{ + "type" => "code_interpreter_logs", + "call_id" => "ci_xxx", + "logs" => "4\n" + } + ] + } + } + } + + assert [%ReqLLM.StreamChunk{type: :meta, metadata: metadata}] = + ResponsesAPI.decode_stream_event(completed, model) + + assert metadata.terminal? == true + assert [call, logs] = get_in(metadata.provider_meta, ["code_interpreter", "items"]) + assert call["type"] == "code_interpreter_call" + assert logs["type"] == "code_interpreter_logs" + end + test "stateful decoding avoids duplicate completed output items", %{model: model} do added = %{ data: %{