diff --git a/tools/gen-ai/mock-server/src/genai_mock_server/anthropic.py b/tools/gen-ai/mock-server/src/genai_mock_server/anthropic.py index ed3100e2..62ef9dee 100644 --- a/tools/gen-ai/mock-server/src/genai_mock_server/anthropic.py +++ b/tools/gen-ai/mock-server/src/genai_mock_server/anthropic.py @@ -82,16 +82,61 @@ } +def _current_turn(messages): + """Messages since the last user message that is not a tool result. + + A tool called in an earlier turn must not stop the model from calling it + again when the user asks a new question. + """ + last_user = -1 + for index, message in enumerate(messages): + if message.get("role") != "user": + continue + content = message.get("content") + if isinstance(content, list) and any( + isinstance(block, dict) and block.get("type") == "tool_result" + for block in content + ): + continue + last_user = index + return messages[last_user + 1 :] + + def _has_tool_result(body): for message in body.get("messages", []): content = message.get("content") if isinstance(content, list): for block in content: - if isinstance(block, dict) and block.get("type") == "tool_result": + if ( + isinstance(block, dict) + and block.get("type") == "tool_result" + ): return True return False +def _get_anthropic_tool_info(messages): + called_names = set() + call_ids = [] + for message in messages: + content = message.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict): + if block.get("type") == "tool_use": + call_id = block.get("id") + if call_id: + call_ids.append(call_id) + name = block.get("name") + if name: + called_names.add(name) + elif block.get("type") == "tool_result": + call_id = block.get("tool_use_id") + if call_id: + call_ids.append(call_id) + return called_names, call_ids + + def _sse_event(event_type, data): return f"event: {event_type}\ndata: {json.dumps(data)}\n\n" @@ -170,20 +215,41 @@ def messages(): context_management = body.get("context_management") or {} edits = context_management.get("edits") or [] - if any(edit.get("type") == "compact_20260112" for edit in edits if isinstance(edit, dict)): + if any( + edit.get("type") == "compact_20260112" + for edit in edits + if isinstance(edit, dict) + ): resp = copy.deepcopy(MESSAGE_COMPACTION_RESPONSE) resp["model"] = body.get("model", resp["model"]) return resp - if body.get("tools") and not _has_tool_result(body): - resp = copy.deepcopy(MESSAGE_TOOL_USE_RESPONSE) - resp["model"] = body.get("model", resp["model"]) - tool = body.get("tools", [{}])[0] - tool_name = tool.get("name") - if tool_name: - resp["content"][0]["name"] = tool_name - resp["content"][0]["input"] = mock_tool_arguments(tool) - return resp + if body.get("tools"): + tools = body.get("tools") + first_tool = tools[0] if tools else {} + tool_name = first_tool.get("name") + messages = body.get("messages", []) + _, call_ids = _get_anthropic_tool_info(messages) + turn = _current_turn(messages) + called_names, _ = _get_anthropic_tool_info(turn) + if not _has_tool_result({"messages": turn}): + should_call = True + elif called_names: + should_call = bool(tool_name) and tool_name not in called_names + else: + # A result with no call to attribute it to: answer rather than loop. + should_call = False + + if should_call: + call_idx = len(set(call_ids)) + 1 + call_id = f"toolu_mock_{call_idx:03d}" + resp = copy.deepcopy(MESSAGE_TOOL_USE_RESPONSE) + resp["model"] = body.get("model", resp["model"]) + if tool_name: + resp["content"][0]["name"] = tool_name + resp["content"][0]["id"] = call_id + resp["content"][0]["input"] = mock_tool_arguments(first_tool) + return resp resp = copy.deepcopy(MESSAGE_RESPONSE) resp["model"] = body.get("model", resp["model"]) diff --git a/tools/gen-ai/mock-server/src/genai_mock_server/bedrock.py b/tools/gen-ai/mock-server/src/genai_mock_server/bedrock.py index 9ecd7dd0..98ee416e 100644 --- a/tools/gen-ai/mock-server/src/genai_mock_server/bedrock.py +++ b/tools/gen-ai/mock-server/src/genai_mock_server/bedrock.py @@ -1,5 +1,6 @@ """AWS Bedrock-compatible endpoints.""" +import base64 import copy import json @@ -75,7 +76,9 @@ def _tool_to_call(body): ] if not specifications: return None - chosen = ((tool_config.get("toolChoice") or {}).get("tool") or {}).get("name") + chosen = ((tool_config.get("toolChoice") or {}).get("tool") or {}).get( + "name" + ) for specification in specifications: if specification["name"] == chosen: return specification @@ -96,14 +99,23 @@ def _stream_converse(): events = [] events.append(("messageStart", {"role": "assistant"})) for word in ["This ", "is ", "a ", "mock ", "streamed ", "response."]: - events.append(("contentBlockDelta", {"delta": {"text": word}, "contentBlockIndex": 0})) + events.append( + ( + "contentBlockDelta", + {"delta": {"text": word}, "contentBlockIndex": 0}, + ) + ) events.append(("contentBlockStop", {"contentBlockIndex": 0})) events.append(("messageStop", {"stopReason": "end_turn"})) events.append( ( "metadata", { - "usage": {"inputTokens": 25, "outputTokens": 6, "totalTokens": 31}, + "usage": { + "inputTokens": 25, + "outputTokens": 6, + "totalTokens": 31, + }, "metrics": {"latencyMs": 100}, }, ) @@ -125,14 +137,91 @@ def bedrock_converse(model_id): @bp.route("/model//converse-stream", methods=["POST"]) def bedrock_converse_stream(model_id): - return Response(_stream_converse(), mimetype="application/vnd.amazon.eventstream") + return Response( + _stream_converse(), mimetype="application/vnd.amazon.eventstream" + ) @bp.route("/model//invoke", methods=["POST"]) def bedrock_invoke(model_id): - """Handle Bedrock InvokeModel — used for Titan Embeddings.""" + """Handle Bedrock InvokeModel.""" + if "embed" in model_id: + resp = { + "embedding": [0.001] * 256, + "inputTextTokenCount": 8, + } + headers = { + "x-amzn-bedrock-input-token-count": "8", + "x-amzn-bedrock-content-type": "application/json", + } + return Response( + json.dumps(resp), mimetype="application/json", headers=headers + ) + resp = { - "embedding": [0.001] * 256, - "inputTextTokenCount": 8, + "inputTextTokenCount": 5, + "results": [ + { + "tokenCount": 10, + "outputText": "This is a response from the mock server.", + "completionReason": "FINISH", + } + ], } - return Response(json.dumps(resp), mimetype="application/json") + headers = { + "x-amzn-bedrock-input-token-count": "5", + "x-amzn-bedrock-output-token-count": "10", + "x-amzn-bedrock-content-type": "application/json", + } + return Response( + json.dumps(resp), mimetype="application/json", headers=headers + ) + + +def _stream_invoke(): + """Yield Bedrock InvokeModelWithResponseStream event-stream chunks in binary format.""" + # Titan streams `totalOutputTextTokenCount`, not the `tokenCount` its + # non-streaming response carries, and only fills it on the final chunk. + chunks = [ + { + "outputText": "This is ", + "index": 0, + "totalOutputTextTokenCount": None, + "completionReason": None, + "inputTextTokenCount": 5, + }, + { + "outputText": "a test", + "index": 0, + "totalOutputTextTokenCount": 10, + "completionReason": "FINISH", + "inputTextTokenCount": 5, + "amazon-bedrock-invocationMetrics": { + "inputTokenCount": 5, + "outputTokenCount": 10, + "firstByteLatency": 100, + "invocationLatency": 200, + }, + }, + ] + for chunk in chunks: + raw_bytes = json.dumps(chunk).encode("utf-8") + payload = json.dumps( + {"bytes": base64.b64encode(raw_bytes).decode("ascii")} + ).encode("utf-8") + yield encode_aws_event_stream_message("chunk", payload) + + +@bp.route( + "/model//invoke-with-response-stream", methods=["POST"] +) +def bedrock_invoke_stream(model_id): + """Handle Bedrock InvokeModelWithResponseStream.""" + headers = { + "x-amzn-bedrock-content-type": "application/json", + } + return Response( + _stream_invoke(), + mimetype="application/vnd.amazon.eventstream", + headers=headers, + ) diff --git a/tools/gen-ai/mock-server/src/genai_mock_server/openai.py b/tools/gen-ai/mock-server/src/genai_mock_server/openai.py index 9f0a7f3e..89bc1999 100644 --- a/tools/gen-ai/mock-server/src/genai_mock_server/openai.py +++ b/tools/gen-ai/mock-server/src/genai_mock_server/openai.py @@ -123,7 +123,10 @@ # OpenAI breaks out audio (and cached) tokens within the prompt total. "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "completion_tokens": 20, - "completion_tokens_details": {"audio_tokens": 0, "reasoning_tokens": 0}, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 0, + }, "total_tokens": 60, }, } @@ -205,7 +208,10 @@ def _has_audio_input(body): content = message.get("content") if isinstance(content, list): for part in content: - if isinstance(part, dict) and part.get("type") == "input_audio": + if ( + isinstance(part, dict) + and part.get("type") == "input_audio" + ): return True return False @@ -228,7 +234,49 @@ def _chat_audio_response(body): return response -def _responses_tool_call_response(body): +def _get_offered_tool(body): + tools = body.get("tools") or [] + if not tools: + return None, None + tool = tools[0] + function = tool.get("function", tool) + name = function.get("name") or tool.get("name") + return tool, name + + +def _current_turn(messages): + """Messages since the last user message. + + A tool called in an earlier turn must not stop the model from calling it + again when the user asks a new question. + """ + last_user = -1 + for index, message in enumerate(messages): + if message.get("role") == "user": + last_user = index + return messages[last_user + 1 :] + + +def _get_called_tool_info(messages): + called_names = set() + call_ids = [] + for m in messages: + if m.get("role") == "assistant": + for tc in m.get("tool_calls", []): + call_id = tc.get("id") + if call_id: + call_ids.append(call_id) + name = tc.get("function", {}).get("name") + if name: + called_names.add(name) + elif m.get("role") == "tool": + call_id = m.get("tool_call_id") + if call_id: + call_ids.append(call_id) + return called_names, call_ids + + +def _responses_tool_call_response(body, call_index=1): response = copy.deepcopy(RESPONSES_RESPONSE) response["id"] = "resp-mock-tool-001" response["model"] = body.get("model", response["model"]) @@ -238,8 +286,8 @@ def _responses_tool_call_response(body): response["output"] = [ { "type": "function_call", - "id": "fc_mock_001", - "call_id": "call_mock_001", + "id": f"fc_mock_{call_index:03d}", + "call_id": f"call_mock_{call_index:03d}", "name": tool_name or "get_weather", "arguments": json.dumps(mock_tool_arguments(tool)), "status": "completed", @@ -252,7 +300,10 @@ def _mock_chat_content(body, message_text): # CrewAI converter retry, recognised by its schema-conversion system prompt # (crewai/translations/en.json, formatted_task_instructions): answer with a # PlannerTaskPydanticOutput-shaped body so the conversion succeeds. - if "Format your final answer according to the following OpenAPI schema" in message_text: + if ( + "Format your final answer according to the following OpenAPI schema" + in message_text + ): return json.dumps( { "list_of_plans_per_task": [ @@ -305,7 +356,10 @@ def _mock_chat_content(body, message_text): if response_format.get("type") != "json_object": return "This is a response from the mock server." - if "Relevance-Judge" in message_text or "Relevance Evaluator" in message_text: + if ( + "Relevance-Judge" in message_text + or "Relevance Evaluator" in message_text + ): return json.dumps( { "explanation": "The response directly answers the user's question and stays fully on topic.", @@ -340,7 +394,9 @@ def _text_protocol_tool_call(body, message_text): tools = [] # The instructions mention an empty pair before the real # one, so every section is scanned rather than just the first. - for section in re.findall(r"(.*?)", message_text, re.DOTALL): + for section in re.findall( + r"(.*?)", message_text, re.DOTALL + ): for line in section.strip().splitlines(): try: tools.append(json.loads(line)) @@ -372,18 +428,30 @@ def _text_protocol_tool_call(body, message_text): ) -def _wants_tool_call(body): +def _should_call_tool(body): """Whether this request should be answered with a call to its first tool. - Offered tools and no tool result yet, which is the same rule the - non-streaming path follows so a framework sees the same exchange either - way. + Offered tools and, in the current turn, no result yet for that tool. The + streaming and non-streaming paths share the rule so a framework sees the + same exchange either way. """ if not body.get("tools"): return False - return not any( - message.get("role") == "tool" for message in body.get("messages", []) - ) + tool, tool_name = _get_offered_tool(body) + if not tool or not tool_name: + return False + turn = _current_turn(body.get("messages", [])) + called_names, _ = _get_called_tool_info(turn) + has_tool_result = any(m.get("role") == "tool" for m in turn) + + if not has_tool_result: + return True + + # A result with no call to attribute it to: answer rather than loop. + if called_names: + return tool_name not in called_names + + return False def _stream_tool_call(body, model, chunk_id): @@ -398,8 +466,17 @@ def _stream_tool_call(body, model, chunk_id): name = function.get("name") or "get_weather" arguments = json.dumps(mock_tool_arguments(tool)) + messages = body.get("messages", []) + _, call_ids = _get_called_tool_info(messages) + call_idx = len(set(call_ids)) + 1 + call_id = f"call_mock_{call_idx:03d}" + for delta in ( - {"id": "call_mock_001", "type": "function", "function": {"name": name, "arguments": ""}}, + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": ""}, + }, {"function": {"arguments": arguments}}, ): yield sse( @@ -424,7 +501,9 @@ def _stream_tool_call(body, model, chunk_id): "object": "chat.completion.chunk", "created": 1700000000, "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + "choices": [ + {"index": 0, "delta": {}, "finish_reason": "tool_calls"} + ], "usage": { "prompt_tokens": 50, "completion_tokens": 20, @@ -448,11 +527,17 @@ def _stream_chat(body): "created": 1700000000, "model": model, "service_tier": _served_service_tier(body), - "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}], + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": None, + } + ], } ) - if _wants_tool_call(body): + if _should_call_tool(body): yield from _stream_tool_call(body, model, chunk_id) return @@ -474,7 +559,13 @@ def _stream_chat(body): "object": "chat.completion.chunk", "created": 1700000000, "model": model, - "choices": [{"index": 0, "delta": {"content": word}, "finish_reason": None}], + "choices": [ + { + "index": 0, + "delta": {"content": word}, + "finish_reason": None, + } + ], } ) @@ -484,7 +575,9 @@ def _stream_chat(body): "object": "chat.completion.chunk", "created": 1700000000, "model": model, - "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "delta": {"content": ""}, "finish_reason": "stop"} + ], "usage": { "prompt_tokens": 25, "completion_tokens": 6, @@ -498,7 +591,9 @@ def _stream_chat(body): @bp.route("/v1/chat/completions", methods=["POST"]) @bp.route("/openai/v1/chat/completions", methods=["POST"]) -@bp.route("/openai/deployments//chat/completions", methods=["POST"]) +@bp.route( + "/openai/deployments//chat/completions", methods=["POST"] +) @bp.route("/chat/completions", methods=["POST"]) def chat_completions(deployment=None): body = request.get_json(silent=True) or {} @@ -507,23 +602,29 @@ def chat_completions(deployment=None): return Response(_stream_chat(body), mimetype="text/event-stream") message_text = "\n".join( - message.get("content", "") for message in body.get("messages", []) if isinstance(message.get("content"), str) + message.get("content", "") + for message in body.get("messages", []) + if isinstance(message.get("content"), str) ) - # Offered tools but no tool result yet: call the tool, else answer. - if body.get("tools"): + # Offered tools: call the offered tool unless it was already called in this conversation. + if _should_call_tool(body): messages = body.get("messages", []) - has_tool_result = any(m.get("role") == "tool" for m in messages) - if not has_tool_result: + tool, tool_name = _get_offered_tool(body) + called_names, call_ids = _get_called_tool_info(messages) + if tool: + call_idx = len(set(call_ids)) + 1 + call_id = f"call_mock_{call_idx:03d}" resp = copy.deepcopy(CHAT_TOOL_CALL_RESPONSE) resp["model"] = body.get("model", resp["model"]) - tool = body.get("tools", [{}])[0] - tool_name = tool.get("function", {}).get("name") + resp["choices"][0]["message"]["tool_calls"][0]["id"] = call_id if tool_name: - resp["choices"][0]["message"]["tool_calls"][0]["function"]["name"] = tool_name - resp["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = json.dumps( - mock_tool_arguments(tool) - ) + resp["choices"][0]["message"]["tool_calls"][0]["function"][ + "name" + ] = tool_name + resp["choices"][0]["message"]["tool_calls"][0]["function"][ + "arguments" + ] = json.dumps(mock_tool_arguments(tool)) resp["service_tier"] = _served_service_tier(body) return resp @@ -554,7 +655,9 @@ def chat_completions(deployment=None): # convert_with_instructions and a third LLM round-trip. resp = copy.deepcopy(CHAT_RESPONSE) resp["model"] = body.get("model", resp["model"]) - resp["choices"][0]["message"]["content"] = "I drafted this plan but it is not in the requested schema." + resp["choices"][0]["message"]["content"] = ( + "I drafted this plan but it is not in the requested schema." + ) resp["service_tier"] = _served_service_tier(body) return resp @@ -618,7 +721,9 @@ def responses(): body = request.get_json(silent=True) or {} raw_request_input = body.get("input") if isinstance(raw_request_input, list): - request_input = [item for item in raw_request_input if isinstance(item, dict)] + request_input = [ + item for item in raw_request_input if isinstance(item, dict) + ] else: request_input = [] # Call the first offered tool unless it has already been called in this @@ -634,16 +739,27 @@ def responses(): for item in request_input if item.get("type") == "function_call" } - if body.get("tools") and "agent_reference" not in body and not (offered & called): - return _responses_tool_call_response(body) + if ( + body.get("tools") + and "agent_reference" not in body + and not (offered & called) + ): + call_index = len(called) + 1 + return _responses_tool_call_response(body, call_index=call_index) resp = copy.deepcopy(RESPONSES_RESPONSE) resp["model"] = body.get("model", resp["model"]) if body.get("instructions") is not None: resp["instructions"] = body["instructions"] context_management = body.get("context_management") or [] - if any(item.get("type") == "compaction" for item in context_management if isinstance(item, dict)): - resp["output"][0]["content"][0]["text"] = "Great question. Here is Jevons Paradox in simple terms." + if any( + item.get("type") == "compaction" + for item in context_management + if isinstance(item, dict) + ): + resp["output"][0]["content"][0]["text"] = ( + "Great question. Here is Jevons Paradox in simple terms." + ) resp["output"].append( { "type": "compaction", @@ -671,8 +787,20 @@ def _stream_response(response): in_progress["status"] = "in_progress" in_progress["output"] = [] in_progress["usage"] = None - yield sse({"type": "response.created", "sequence_number": 0, "response": in_progress}) - yield sse({"type": "response.in_progress", "sequence_number": 1, "response": in_progress}) + yield sse( + { + "type": "response.created", + "sequence_number": 0, + "response": in_progress, + } + ) + yield sse( + { + "type": "response.in_progress", + "sequence_number": 1, + "response": in_progress, + } + ) item = copy.deepcopy(response["output"][0]) sequence = 2 @@ -731,7 +859,11 @@ def _stream_response(response): "item_id": item["id"], "output_index": 0, "content_index": 0, - "part": {"type": "output_text", "text": text, "annotations": []}, + "part": { + "type": "output_text", + "text": text, + "annotations": [], + }, } ) sequence += 1 @@ -744,7 +876,13 @@ def _stream_response(response): } ) sequence += 1 - yield sse({"type": "response.completed", "sequence_number": sequence, "response": response}) + yield sse( + { + "type": "response.completed", + "sequence_number": sequence, + "response": response, + } + ) def _stream_create(response): @@ -757,8 +895,20 @@ def _stream_create(response): in_progress["status"] = "in_progress" in_progress["output"] = [] in_progress["usage"] = None - yield sse({"type": "response.created", "sequence_number": 0, "response": in_progress}) - yield sse({"type": "response.in_progress", "sequence_number": 1, "response": in_progress}) + yield sse( + { + "type": "response.created", + "sequence_number": 0, + "response": in_progress, + } + ) + yield sse( + { + "type": "response.in_progress", + "sequence_number": 1, + "response": in_progress, + } + ) @bp.route("/v1/responses/", methods=["GET"]) @@ -796,13 +946,18 @@ def retrieve_response(response_id): } }, 400 starting_after = request.args.get("starting_after") - return Response(_stream_retrieve(stored, starting_after), mimetype="text/event-stream") + return Response( + _stream_retrieve(stored, starting_after), + mimetype="text/event-stream", + ) return dict(stored) def _stream_retrieve(response, starting_after): """Yield SSE events resuming a stored response stream after `starting_after`.""" - sequence_number = int(starting_after) + 1 if starting_after is not None else 0 + sequence_number = ( + int(starting_after) + 1 if starting_after is not None else 0 + ) yield sse( { "type": "response.completed", diff --git a/tools/gen-ai/mock-server/tests/test_mock_server.py b/tools/gen-ai/mock-server/tests/test_mock_server.py index f0ee5b98..c4ae32e8 100644 --- a/tools/gen-ai/mock-server/tests/test_mock_server.py +++ b/tools/gen-ai/mock-server/tests/test_mock_server.py @@ -8,8 +8,10 @@ 200. """ +import base64 import json import re +import struct import pytest @@ -21,7 +23,10 @@ "openai-chat", "post", "/v1/chat/completions", - {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + }, ), ( "openai-chat-json-schema", @@ -110,17 +115,42 @@ "/model/amazon.titan-text-express-v1/converse", {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}, ), + ( + "bedrock-invoke", + "post", + "/model/amazon.titan-text-express-v1/invoke", + {"inputText": "hi"}, + ), + ( + "bedrock-invoke-embeddings", + "post", + "/model/amazon.titan-embed-text-v1/invoke", + {"inputText": "hi"}, + ), + ( + "bedrock-invoke-stream", + "post", + "/model/amazon.titan-text-express-v1/invoke-with-response-stream", + {"inputText": "hi"}, + ), ( "cohere", "post", "/v2/chat", - {"model": "command-r", "messages": [{"role": "user", "content": "hi"}]}, + { + "model": "command-r", + "messages": [{"role": "user", "content": "hi"}], + }, ), ( "cohere-embed", "post", "/v2/embed", - {"model": "embed-v4.0", "texts": ["hi", "there"], "input_type": "search_document"}, + { + "model": "embed-v4.0", + "texts": ["hi", "there"], + "input_type": "search_document", + }, ), ( "mistral-chat", @@ -164,11 +194,21 @@ # Resource-creating endpoints mint a fresh id per call, so only the shape is # stable. Kept separate rather than loosening the assertion above. CREATE_ENDPOINTS = [ - ("anthropic-agents", "post", "/v1/agents", {"model": "claude-sonnet-4-20250514"}), + ( + "anthropic-agents", + "post", + "/v1/agents", + {"model": "claude-sonnet-4-20250514"}, + ), ("bedrock-agent", "put", "/agents/", {"agentName": "mock-agent"}), ("bedrock-agentcore", "post", "/memories/create", {"name": "mock-memory"}), ("openai-assistants", "post", "/v1/assistants", {"model": "gpt-4o-mini"}), - ("mistral-agents", "post", "/mistral/v1/agents", {"model": "mistral-medium-latest"}), + ( + "mistral-agents", + "post", + "/mistral/v1/agents", + {"model": "mistral-medium-latest"}, + ), ] @@ -200,7 +240,9 @@ def test_endpoint_answers_deterministically(client, method, path, body): [case[1:] for case in CREATE_ENDPOINTS], ids=[case[0] for case in CREATE_ENDPOINTS], ) -def test_create_endpoint_answers_with_a_stable_shape(client, method, path, body): +def test_create_endpoint_answers_with_a_stable_shape( + client, method, path, body +): first = getattr(client, method)(path, json=body) assert first.status_code < 300, first.data @@ -212,7 +254,10 @@ def test_create_endpoint_answers_with_a_stable_shape(client, method, path, body) def test_chat_echoes_the_requested_model(client): response = client.post( "/v1/chat/completions", - json={"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}]}, + json={ + "model": "gpt-5", + "messages": [{"role": "user", "content": "hi"}], + }, ) body = response.json assert body["model"] == "gpt-5" @@ -221,6 +266,320 @@ def test_chat_echoes_the_requested_model(client): assert body["service_tier"] == "default" +def test_chat_multi_turn_tool_call_generates_unique_ids(client): + """Multi-turn agent handoff gets unique tool call IDs per turn and answers once all tools replied.""" + # Turn 1: user asks triage agent + r1 = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "weather in Seattle?"}], + "tools": [ + { + "type": "function", + "function": {"name": "transfer_to_specialist"}, + } + ], + }, + ) + tc1 = r1.json["choices"][0]["message"]["tool_calls"][0] + assert tc1["id"] == "call_mock_001" + assert tc1["function"]["name"] == "transfer_to_specialist" + + # Turn 2: handoff to specialist offering get_weather + r2 = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + {"role": "assistant", "tool_calls": [tc1]}, + { + "role": "tool", + "tool_call_id": "call_mock_001", + "content": "transferred", + }, + ], + "tools": [ + {"type": "function", "function": {"name": "get_weather"}} + ], + }, + ) + tc2 = r2.json["choices"][0]["message"]["tool_calls"][0] + assert tc2["id"] == "call_mock_002" + assert tc2["function"]["name"] == "get_weather" + + # Turn 3: get_weather replied -> final answer + r3 = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + {"role": "assistant", "tool_calls": [tc1]}, + { + "role": "tool", + "tool_call_id": "call_mock_001", + "content": "transferred", + }, + {"role": "assistant", "tool_calls": [tc2]}, + { + "role": "tool", + "tool_call_id": "call_mock_002", + "content": "70 degrees", + }, + ], + "tools": [ + {"type": "function", "function": {"name": "get_weather"}} + ], + }, + ) + assert "tool_calls" not in r3.json["choices"][0]["message"] + assert r3.json["choices"][0]["finish_reason"] == "stop" + + +def test_anthropic_multi_turn_tool_call_generates_unique_ids(client): + """Anthropic multi-turn tool calling gets unique tool IDs per turn.""" + # Turn 1 + r1 = client.post( + "/v1/messages", + json={ + "model": "claude-sonnet-4-20250514", + "messages": [{"role": "user", "content": "weather?"}], + "tools": [ + {"name": "transfer_agent", "input_schema": {"type": "object"}} + ], + }, + ) + block1 = r1.json["content"][0] + assert block1["type"] == "tool_use" + assert block1["id"] == "toolu_mock_001" + assert block1["name"] == "transfer_agent" + + # Turn 2 + r2 = client.post( + "/v1/messages", + json={ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": [block1]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_mock_001", + "content": "transferred", + } + ], + }, + ], + "tools": [ + {"name": "get_weather", "input_schema": {"type": "object"}} + ], + }, + ) + block2 = r2.json["content"][0] + assert block2["type"] == "tool_use" + assert block2["id"] == "toolu_mock_002" + assert block2["name"] == "get_weather" + + # Turn 3 + r3 = client.post( + "/v1/messages", + json={ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": [block1]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_mock_001", + "content": "transferred", + } + ], + }, + {"role": "assistant", "content": [block2]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_mock_002", + "content": "70 degrees", + } + ], + }, + ], + "tools": [ + {"name": "get_weather", "input_schema": {"type": "object"}} + ], + }, + ) + assert r3.json["content"][0]["type"] == "text" + assert r3.json["stop_reason"] == "end_turn" + + +def test_chat_calls_the_same_tool_again_in_a_later_turn(client): + """A completed call does not exhaust the tool: a new user turn calls it again.""" + response = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_mock_001", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_mock_001", + "content": "70 degrees", + }, + {"role": "assistant", "content": "It is 70 degrees."}, + {"role": "user", "content": "and in Portland?"}, + ], + "tools": [ + {"type": "function", "function": {"name": "get_weather"}} + ], + }, + ) + call = response.json["choices"][0]["message"]["tool_calls"][0] + assert call["function"]["name"] == "get_weather" + assert call["id"] == "call_mock_002" + + +def test_anthropic_calls_the_same_tool_again_in_a_later_turn(client): + response = client.post( + "/v1/messages", + json={ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_mock_001", + "name": "get_weather", + "input": {}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_mock_001", + "content": "70 degrees", + } + ], + }, + {"role": "assistant", "content": "It is 70 degrees."}, + {"role": "user", "content": "and in Portland?"}, + ], + "tools": [ + {"name": "get_weather", "input_schema": {"type": "object"}} + ], + }, + ) + block = response.json["content"][0] + assert block["type"] == "tool_use" + assert block["name"] == "get_weather" + assert block["id"] == "toolu_mock_002" + + +def test_anthropic_answers_when_a_tool_result_has_no_matching_call(client): + """A result the mock cannot attribute to a call ends the exchange, as on the OpenAI side.""" + response = client.post( + "/v1/messages", + json={ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_x", + "content": "70 degrees", + } + ], + }, + ], + "tools": [ + {"name": "get_weather", "input_schema": {"type": "object"}} + ], + }, + ) + assert response.json["content"][0]["type"] == "text" + assert response.json["stop_reason"] == "end_turn" + + +def test_streaming_chat_numbers_tool_calls_across_turns(client): + """The streamed path mints the same sequential ids the non-streamed one does.""" + response = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "weather in Seattle?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_mock_001", + "type": "function", + "function": { + "name": "transfer_to_specialist", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_mock_001", + "content": "transferred", + }, + ], + "tools": [ + {"type": "function", "function": {"name": "get_weather"}} + ], + "stream": True, + }, + ) + deltas = [ + json.loads(line[len("data: ") :]) + for line in response.get_data(as_text=True).splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ] + calls = [ + call + for delta in deltas + for call in delta["choices"][0]["delta"].get("tool_calls", []) + ] + assert calls[0]["id"] == "call_mock_002" + assert calls[0]["function"]["name"] == "get_weather" + + def test_chat_returns_a_tool_call_when_tools_are_offered(client): response = client.post( "/v1/chat/completions", @@ -325,6 +684,72 @@ def test_bedrock_converse_answers_when_no_tool_is_offered(client): assert response.json["stopReason"] == "end_turn" +def test_bedrock_invoke_headers(client): + response = client.post( + "/model/amazon.titan-text-express-v1/invoke", + json={"inputText": "hi"}, + ) + assert response.status_code == 200 + assert response.headers["x-amzn-bedrock-input-token-count"] == "5" + assert response.headers["x-amzn-bedrock-output-token-count"] == "10" + assert ( + response.headers["x-amzn-bedrock-content-type"] == "application/json" + ) + assert response.json["inputTextTokenCount"] == 5 + assert ( + response.json["results"][0]["outputText"] + == "This is a response from the mock server." + ) + + +def test_bedrock_invoke_embeddings_report_input_tokens(client): + response = client.post( + "/model/amazon.titan-embed-text-v1/invoke", + json={"inputText": "hi"}, + ) + assert response.headers["x-amzn-bedrock-input-token-count"] == "8" + assert response.json["inputTextTokenCount"] == 8 + + +def test_bedrock_invoke_stream_returns_eventstream(client): + response = client.post( + "/model/amazon.titan-text-express-v1/invoke-with-response-stream", + json={"inputText": "hi"}, + ) + assert response.status_code == 200 + assert response.mimetype == "application/vnd.amazon.eventstream" + assert ( + response.headers["x-amzn-bedrock-content-type"] == "application/json" + ) + + data = response.data + chunks = [] + offset = 0 + while offset < len(data): + total_length, headers_length = struct.unpack_from("!II", data, offset) + payload = data[ + offset + 12 + headers_length : offset + total_length - 4 + ] + frame = json.loads(payload.decode("utf-8")) + chunk_json = json.loads( + base64.b64decode(frame["bytes"]).decode("utf-8") + ) + chunks.append(chunk_json) + offset += total_length + + assert len(chunks) == 2 + assert chunks[0]["outputText"] == "This is " + # Titan only fills the running token count on the last chunk. + assert chunks[0]["totalOutputTextTokenCount"] is None + assert chunks[0]["completionReason"] is None + assert chunks[1]["outputText"] == "a test" + assert chunks[1]["totalOutputTextTokenCount"] == 10 + assert chunks[1]["completionReason"] == "FINISH" + metrics = chunks[1]["amazon-bedrock-invocationMetrics"] + assert metrics["inputTokenCount"] == 5 + assert metrics["outputTokenCount"] == 10 + + def test_embeddings_treat_token_ids_as_one_input(client): response = client.post( "/v1/embeddings", @@ -417,7 +842,10 @@ def test_json_schema_follows_refs_and_skips_null_branches(client): {"type": "string"}, ] }, - "issued": {"type": "string", "format": "date-time"}, + "issued": { + "type": "string", + "format": "date-time", + }, }, }, }, @@ -447,7 +875,9 @@ def test_json_schema_cuts_off_a_self_referencing_model(client): "$defs": { "Node": { "type": "object", - "properties": {"child": {"$ref": "#/$defs/Node"}}, + "properties": { + "child": {"$ref": "#/$defs/Node"} + }, } }, "properties": {"root": {"$ref": "#/$defs/Node"}}, @@ -533,11 +963,18 @@ def test_chat_answers_a_json_schema_with_a_matching_object(client): "type": "array", "items": { "type": "object", - "properties": {"summary": {"type": "string"}}, + "properties": { + "summary": {"type": "string"} + }, }, }, }, - "required": ["location", "temperature", "conditions", "days"], + "required": [ + "location", + "temperature", + "conditions", + "days", + ], }, }, }, @@ -601,9 +1038,9 @@ def test_streaming_chat_calls_an_offered_tool(client): "get_current_weather", None, ] - assert json.loads("".join(call["function"]["arguments"] for call in calls)) == { - "location": "Seattle" - } + assert json.loads( + "".join(call["function"]["arguments"] for call in calls) + ) == {"location": "Seattle"} assert deltas[-1]["choices"][0]["finish_reason"] == "tool_calls" @@ -616,9 +1053,18 @@ def test_streaming_chat_answers_once_the_tool_has_replied(client): "messages": [ {"role": "user", "content": "weather in Seattle?"}, {"role": "assistant", "tool_calls": []}, - {"role": "tool", "content": "70 degrees", "tool_call_id": "call_mock_001"}, + { + "role": "tool", + "content": "70 degrees", + "tool_call_id": "call_mock_001", + }, + ], + "tools": [ + { + "type": "function", + "function": {"name": "get_current_weather"}, + } ], - "tools": [{"type": "function", "function": {"name": "get_current_weather"}}], "stream": True, }, ) @@ -670,7 +1116,12 @@ def test_mistral_chat_answers_once_the_tool_has_replied(client): "tool_call_id": "callmock1", }, ], - "tools": [{"type": "function", "function": {"name": "get_current_weather"}}], + "tools": [ + { + "type": "function", + "function": {"name": "get_current_weather"}, + } + ], }, ) assert response.json["choices"][0]["message"]["tool_calls"] is None @@ -704,7 +1155,9 @@ def test_mistral_chat_streams_the_same_answer_it_would_return(client): "messages": [{"role": "user", "content": "hi"}], } complete = client.post("/mistral/v1/chat/completions", json=body) - streamed = client.post("/mistral/v1/chat/completions", json={**body, "stream": True}) + streamed = client.post( + "/mistral/v1/chat/completions", json={**body, "stream": True} + ) chunks = [ json.loads(line[len("data: ") :]) for line in streamed.get_data(as_text=True).splitlines() @@ -761,7 +1214,10 @@ def test_mistral_embeddings_answer_one_vector_per_input(client): # Azure routes the same operation under a deployment path; instrumentations # read the URL, so the alias has to serve the identical body. def test_azure_deployment_path_matches_the_plain_one(client): - body = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]} + body = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + } plain = client.post("/v1/chat/completions", json=body) deployment = client.post( "/openai/deployments/gpt-4o-mini/chat/completions", json=body @@ -810,7 +1266,9 @@ def test_text_protocol_tools_get_a_tool_call_in_the_content(client): content = response.json["choices"][0]["message"]["content"] assert content.startswith("") - call = json.loads(content.removeprefix("").removesuffix("")) + call = json.loads( + content.removeprefix("").removesuffix("") + ) assert call["name"] == "get_weather" assert "location" in call["arguments"] @@ -818,8 +1276,14 @@ def test_text_protocol_tools_get_a_tool_call_in_the_content(client): def test_text_protocol_does_not_loop_once_the_tool_has_answered(client): """A second call after the result must not ask for the tool again.""" answered = _hermes_request( - {"role": "assistant", "content": '\n{"name": "get_weather"}\n'}, - {"role": "user", "content": "\n70 degrees\n"}, + { + "role": "assistant", + "content": '\n{"name": "get_weather"}\n', + }, + { + "role": "user", + "content": "\n70 degrees\n", + }, ) response = client.post("/v1/chat/completions", json=answered) @@ -855,7 +1319,15 @@ def test_chat_reports_audio_tokens_for_audio_input(client): "messages": [ { "role": "user", - "content": [{"type": "input_audio", "input_audio": {"data": "bW9jaw==", "format": "wav"}}], + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "bW9jaw==", + "format": "wav", + }, + } + ], } ], }, @@ -869,7 +1341,9 @@ def test_chat_reports_audio_tokens_for_audio_input(client): def test_responses_reports_cache_write_tokens(client): - response = client.post("/v1/responses", json={"model": "gpt-4o-mini", "input": "hi"}) + response = client.post( + "/v1/responses", json={"model": "gpt-4o-mini", "input": "hi"} + ) details = response.json["usage"]["input_tokens_details"] assert details["cache_write_tokens"] > 0 @@ -886,14 +1360,21 @@ def test_google_reports_a_cached_and_per_modality_breakdown(client): assert usage["cachedContentTokenCount"] > 0 assert [d["modality"] for d in usage["promptTokensDetails"]] == ["TEXT"] assert [d["modality"] for d in usage["cacheTokensDetails"]] == ["TEXT"] - assert [d["modality"] for d in usage["candidatesTokensDetails"]] == ["TEXT"] + assert [d["modality"] for d in usage["candidatesTokensDetails"]] == [ + "TEXT" + ] def test_google_bills_tool_use_tokens_separately_from_the_prompt(client): tools = [{"functionDeclarations": [{"name": "get_weather"}]}] - request = {"contents": [{"role": "user", "parts": [{"text": "weather?"}]}], "tools": tools} + request = { + "contents": [{"role": "user", "parts": [{"text": "weather?"}]}], + "tools": tools, + } - call = client.post("/v1beta/models/gemini-2.0-flash:generateContent", json=request).json + call = client.post( + "/v1beta/models/gemini-2.0-flash:generateContent", json=request + ).json usage = call["usageMetadata"] assert usage["toolUsePromptTokenCount"] > 0 # Tool-use tokens are their own component of the total, not part of the prompt. @@ -910,12 +1391,24 @@ def test_google_answers_in_text_once_the_tool_has_answered(client): answered = { "contents": [ {"role": "user", "parts": [{"text": "weather?"}]}, - {"role": "user", "parts": [{"functionResponse": {"name": "get_weather", "response": {"temp": 70}}}]}, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_weather", + "response": {"temp": 70}, + } + } + ], + }, ], "tools": [{"functionDeclarations": [{"name": "get_weather"}]}], } - body = client.post("/v1beta/models/gemini-2.0-flash:generateContent", json=answered).json + body = client.post( + "/v1beta/models/gemini-2.0-flash:generateContent", json=answered + ).json parts = body["candidates"][0]["content"]["parts"] assert all("functionCall" not in part for part in parts) @@ -934,7 +1427,12 @@ def test_google_breaks_usage_down_by_input_and_output_modality(client): {"text": "describe"}, {"inlineData": blob}, {"inlineData": blob}, - {"inlineData": {"mimeType": "audio/wav", "data": "bW9jaw=="}}, + { + "inlineData": { + "mimeType": "audio/wav", + "data": "bW9jaw==", + } + }, ], } ], @@ -944,13 +1442,21 @@ def test_google_breaks_usage_down_by_input_and_output_modality(client): usage = response.json["usageMetadata"] # One entry per modality, not per part: the two images are summed. - prompt = {d["modality"]: d["tokenCount"] for d in usage["promptTokensDetails"]} + prompt = { + d["modality"]: d["tokenCount"] for d in usage["promptTokensDetails"] + } assert set(prompt) == {"TEXT", "IMAGE", "AUDIO"} assert prompt["IMAGE"] == 2 * 258 assert sum(prompt.values()) == usage["promptTokenCount"] - assert [d["modality"] for d in usage["candidatesTokensDetails"]] == ["TEXT", "IMAGE"] - assert sum(d["tokenCount"] for d in usage["candidatesTokensDetails"]) == usage["candidatesTokenCount"] + assert [d["modality"] for d in usage["candidatesTokensDetails"]] == [ + "TEXT", + "IMAGE", + ] + assert ( + sum(d["tokenCount"] for d in usage["candidatesTokensDetails"]) + == usage["candidatesTokenCount"] + ) assert usage["cachedContentTokenCount"] < usage["promptTokenCount"] @@ -991,7 +1497,12 @@ def test_ollama_chat_answers_once_the_tool_has_replied(client): {"role": "tool", "content": "70 degrees"}, ], "stream": False, - "tools": [{"type": "function", "function": {"name": "get_current_weather"}}], + "tools": [ + { + "type": "function", + "function": {"name": "get_current_weather"}, + } + ], }, ) assert "tool_calls" not in response.json["message"] @@ -1008,8 +1519,13 @@ def test_ollama_streams_newline_delimited_json(client): "stream": True, }, ) - lines = [json.loads(line) for line in response.get_data(as_text=True).splitlines()] - assert [line["done"] for line in lines] == [False] * (len(lines) - 1) + [True] + lines = [ + json.loads(line) + for line in response.get_data(as_text=True).splitlines() + ] + assert [line["done"] for line in lines] == [False] * (len(lines) - 1) + [ + True + ] streamed = "".join(line["message"]["content"] for line in lines).strip() assert streamed == "This is a response from the mock server." assert lines[-1]["eval_count"] == 12 @@ -1041,7 +1557,11 @@ def test_ollama_answers_a_format_request_with_that_schema(client): def test_ollama_embeddings_answer_one_vector_per_input(client): response = client.post( "/api/embed", - json={"model": "nomic-embed-text", "input": ["one", "two"], "dimensions": 64}, + json={ + "model": "nomic-embed-text", + "input": ["one", "two"], + "dimensions": 64, + }, ) assert len(response.json["embeddings"]) == 2 assert len(response.json["embeddings"][0]) == 64 @@ -1085,7 +1605,12 @@ def test_cohere_chat_answers_once_the_tool_has_replied(client): {"role": "user", "content": "weather in Seattle?"}, {"role": "tool", "tool_call_id": "x", "content": "70 degrees"}, ], - "tools": [{"type": "function", "function": {"name": "get_current_weather"}}], + "tools": [ + { + "type": "function", + "function": {"name": "get_current_weather"}, + } + ], }, ) assert "tool_calls" not in response.json["message"]