diff --git a/docs/75-example-coding-agent.md b/docs/75-example-coding-agent.md index 510416ec6..62f79c24b 100644 --- a/docs/75-example-coding-agent.md +++ b/docs/75-example-coding-agent.md @@ -106,6 +106,51 @@ examples/swe_smith/run.sh trainer \ actor_rollout_ref.rollout.n=4 ``` +## Native Qwen3.5 Tool Calling and Thinking + +The harness defaults to one fenced Bash command per turn with thinking disabled. To use +Qwen3.5's native tool format, enable `SMITH_USE_TOOLS`. The harness then provides an +`execute_bash` tool with a string `command` argument, reads structured `tool_calls` from the +response, and returns each observation as a tool message with the matching call ID. Commands +still pass through the same execution checks. + +On **Machine B**, set these entries in `examples/swe_smith/job-template-openai.yaml`: + +```yaml +- name: SMITH_USE_TOOLS + value: "1" +- name: SMITH_ENABLE_THINKING + value: "1" +``` + +These are rollout-container settings: exporting them in the trainer or controller shell does +not pass them to the Jobs. The switches are independent; keep `SMITH_ENABLE_THINKING` at `"0"` +to use native tools without thinking. + +Start the trainer with Qwen3.5's native chat template and the matching vLLM parsers: + +```bash +examples/swe_smith/run.sh trainer \ + actor_rollout_ref.rollout.engine_kwargs.vllm.chat_template=null \ + actor_rollout_ref.rollout.engine_kwargs.vllm.tool_call_parser=qwen3_coder \ + actor_rollout_ref.rollout.engine_kwargs.vllm.reasoning_parser=qwen3 +``` + +Setting `chat_template=null` selects the model's own template instead of the example's Hermes +template. Qwen3.5 uses XML function and parameter tags; vLLM's `qwen3_coder` parser converts +them into the structured calls consumed by the harness. The parser settings follow the +[official Qwen3.5 model card](https://huggingface.co/Qwen/Qwen3.5-9B#serving-qwen35). + +For thinking runs, use native tools as well. The harness retains the returned reasoning in +the assistant history, and tool messages allow Qwen3.5's native template to preserve it across +successive actions. The legacy Bash loop returns observations as user messages, which the +native template treats as new queries and uses to discard earlier reasoning. + +When upgrading an existing deployment, update the checkout on Machine A and restart the +controller with `examples/swe_smith/run.sh controller`. The launcher refreshes the +`swe-smith-agent-scripts` ConfigMap before starting the controller, so new rollout pods use the +updated harness. Apply the Job settings and trainer overrides before starting the next run. + ## Preventing Reward Hacking A coding agent may obtain the reference fix without solving the task, for example by inspecting Git history, downloading upstream source code with `curl` or `wget`, installing the original package with `pip`, or using Python networking libraries such as `urllib`. diff --git a/docs/76-example-coding-agent-moe.md b/docs/76-example-coding-agent-moe.md index b9ca4f381..c8832445d 100644 --- a/docs/76-example-coding-agent-moe.md +++ b/docs/76-example-coding-agent-moe.md @@ -66,5 +66,9 @@ examples/swe_smith/run_moe.sh trainer \ actor_rollout_ref.rollout.n=4 ``` +To enable native Qwen3.5 tools and thinking, follow the [Coding Agent configuration +instructions](75-example-coding-agent.md#native-qwen35-tool-calling-and-thinking), using +`run_moe.sh trainer` with the same vLLM overrides. + The default topology is PP=1, TP=1, EP=4, and ETP=1 with parameter, optimizer, and gradient offload. Treat it as a four-B200 starting point and tune batch sizes for the available memory. diff --git a/examples/swe_smith/agents/smith_agent.py b/examples/swe_smith/agents/smith_agent.py index aff033634..6bf8dd168 100755 --- a/examples/swe_smith/agents/smith_agent.py +++ b/examples/swe_smith/agents/smith_agent.py @@ -109,6 +109,59 @@ Failure to follow these rules will cause your response to be rejected.""" +TOOL_SYSTEM_PROMPT = """\ +You are a helpful assistant that can interact multiple times with a computer +shell to solve programming tasks. Call execute_bash exactly once per turn with +one command (or commands connected with && or ||). Wait for the tool result +before choosing your next command.""" + +BASH_RESPONSE_PREAMBLE = """\ +For each response: +1. Include a THOUGHT section explaining your reasoning and goal. +2. Provide exactly ONE bash command to execute.""" + +BASH_RESPONSE_INSTRUCTIONS = """\ +**CRITICAL REQUIREMENTS:** +- Your response MUST contain EXACTLY ONE bash code block. +- That block MUST contain EXACTLY ONE command (or several joined with && or ||). +- If you include zero or multiple bash blocks, YOUR RESPONSE WILL FAIL. +- Do NOT put several independent commands in separate blocks in one response. +- Directory and environment changes are NOT persistent: every action runs in a new + subshell. Prefix an action with `MY_VAR=value cd /testbed && ...`, or read/write + state from files, if you need persistence. + +Example of a CORRECT response: + +THOUGHT: I need to understand the repository layout first, so let me list the files. + +```bash +ls -la +``` +""" + +TOOL_RESPONSE_INSTRUCTIONS = """\ +For each response, call execute_bash exactly once. Put the shell command in the +command argument, without Markdown fences. For example, call execute_bash with +{"command": "ls -la"}. Do not call multiple tools in one response. + +Directory and environment changes are NOT persistent: every action runs in a new +subshell. Prefix an action with `MY_VAR=value cd /testbed && ...`, or read/write +state from files, if you need persistence.""" + +BASH_TOOL = { + "type": "function", + "function": { + "name": "execute_bash", + "description": "Run one shell command in a fresh subshell in /testbed.", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string", "description": "The shell command to execute."}}, + "required": ["command"], + "additionalProperties": False, + }, + }, +} + INSTANCE_PROMPT = """\ Consider the following PR description: @@ -128,9 +181,7 @@ This is an interactive process: you think and issue ONE command, see \ its result, then think and issue your next command. -For each response: -1. Include a THOUGHT section explaining your reasoning and goal. -2. Provide exactly ONE bash command to execute. +{response_preamble} ## Important Boundaries - MODIFY: regular source files in /testbed (the working directory for every command). @@ -150,23 +201,7 @@ 3. You see the result. 4. You write your next command. -**CRITICAL REQUIREMENTS:** -- Your response MUST contain EXACTLY ONE bash code block. -- That block MUST contain EXACTLY ONE command (or several joined with && or ||). -- If you include zero or multiple bash blocks, YOUR RESPONSE WILL FAIL. -- Do NOT put several independent commands in separate blocks in one response. -- Directory and environment changes are NOT persistent: every action runs in a new - subshell. Prefix an action with `MY_VAR=value cd /testbed && ...`, or read/write - state from files, if you need persistence. - -Example of a CORRECT response: - -THOUGHT: I need to understand the repository layout first, so let me list the files. - -```bash -ls -la -``` - +{response_instructions} ## Environment Details - You have a full Linux shell environment. @@ -185,9 +220,7 @@ ## Submission When your fix is complete, submit with this EXACT command on its own. The system captures your changes to /testbed automatically -- you do NOT need git: -```bash -echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT -``` +{submission_instructions} - git is disabled; do not use it for any purpose. @@ -216,17 +249,63 @@ def __init__(self, n_actions: int) -> None: self.n_actions = n_actions -def parse_action(content: str) -> str: +def parse_action(content: str, *, enable_thinking: bool = False) -> str: """Return the single bash action from a model reply. Raises ``FormatError`` unless exactly one fenced bash block is present. """ + # The model may start inside a prefilled tag. Never execute a + # candidate command from its reasoning, including an unfinished think block. + if enable_thinking or content.lstrip().startswith(""): + content = content.split("", 1)[1] if "" in content else "" actions = _ACTION_RE.findall(content) if len(actions) == 1: return actions[0].strip() raise FormatError(len(actions)) +class UnrenderableToolCall(ValueError): + """Tool arguments cannot be replayed through the server's chat template.""" + + +def parse_tool_action(message: dict[str, Any], finish_reason: str) -> str: + """Validate the one-command contract before executing a native tool call.""" + calls = message.get("tool_calls") or [] + arguments: dict[str, Any] = {} + # vLLM decodes historical arguments as JSON, and Qwen's template iterates + # their keys. Check every call, even when rejecting parallel/unknown tools, + # so error recovery cannot submit history that the server cannot render. + for call in calls: + function = call.get("function") or {} + try: + arguments = json.loads(function.get("arguments", "")) + except (TypeError, json.JSONDecodeError) as exc: + raise UnrenderableToolCall("Tool arguments are not valid JSON.") from exc + if not isinstance(arguments, dict): + raise UnrenderableToolCall("Tool arguments must be a JSON object.") + if finish_reason == "length": + raise ValueError("Your response reached the output token limit. Send one complete execute_bash call.") + if len(calls) != 1: + raise ValueError(f"Expected exactly one execute_bash call, found {len(calls)}.") + call = calls[0] + function = call.get("function") or {} + if call.get("type") != "function" or function.get("name") != "execute_bash": + raise ValueError("Only the execute_bash tool is supported.") + if set(arguments) != {"command"} or not isinstance(arguments["command"], str) or not arguments["command"].strip(): + raise ValueError("execute_bash requires exactly one non-empty string argument named command.") + return arguments["command"].strip() + + +def _append_observation(messages: list[dict[str, Any]], message: dict[str, Any], content: str) -> None: + """Answer every native call, including rejected parallel or malformed calls.""" + calls = message.get("tool_calls") or [] + if calls: + for call in calls: + messages.append({"role": "tool", "tool_call_id": call["id"], "content": content}) + else: + messages.append({"role": "user", "content": content}) + + def is_submission(output: str) -> bool: """True when command output begins with the submit marker.""" stripped = output.lstrip() @@ -301,7 +380,7 @@ def _forbidden_action(action: str) -> str | None: def format_error_message(n_actions: int, finish_reason: str) -> str: """Nudge the model back to the one-bash-block format after a parse failure.""" - if finish_reason in ("length", "tool_calls"): + if finish_reason == "length": return ( f"Your previous response reached the output token limit " f"(finish_reason={finish_reason}) before you produced a complete action, " @@ -727,12 +806,16 @@ def run_agent_loop( max_format_errors: int = 3, gateway_wait_s: float = 600.0, gateway_poll_s: float = 5.0, + use_tools: bool = False, + enable_thinking: bool = False, ) -> tuple[bool, int, int]: """Drive the agent until it submits, errors out, or hits ``max_turns``. Mirrors mini-swe-agent's control flow: each turn we query the model, parse a - single bash action from its free-text reply, run it in a fresh /testbed - subshell, and append a templated observation. + single bash action (a fenced block by default, or a native execute_bash call + with ``use_tools``), run it in a fresh /testbed subshell, and append a + templated observation. ``enable_thinking`` opts into the model's reasoning + mode; reasoning is retained in history but never parsed as a shell action. Returns ``(submitted, turns_used, max_prompt_tokens)`` where ``submitted`` is ``True`` iff the model emitted the submit marker, ``turns_used`` is the number @@ -743,8 +826,24 @@ def run_agent_loop( penalty and ``max_prompt_tokens`` the prompt-length penalty in ``main``. """ messages: list[dict[str, Any]] = [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": INSTANCE_PROMPT.format(problem_statement=problem)}, + {"role": "system", "content": TOOL_SYSTEM_PROMPT if use_tools else SYSTEM_PROMPT}, + { + "role": "user", + "content": INSTANCE_PROMPT.format( + problem_statement=problem, + response_preamble=( + "For each response, call execute_bash exactly once with one command." + if use_tools + else BASH_RESPONSE_PREAMBLE + ), + response_instructions=TOOL_RESPONSE_INSTRUCTIONS if use_tools else BASH_RESPONSE_INSTRUCTIONS, + submission_instructions=( + f'Call execute_bash with {{"command": "echo {SUBMIT_MARKER}"}}.' + if use_tools + else f"```bash\necho {SUBMIT_MARKER}\n```" + ), + ), + }, ] n_format_errors = 0 turns_used = 0 @@ -756,7 +855,9 @@ def run_agent_loop( paused_for = 0.0 while True: try: - content, finish_reason, prompt_tokens = _query(client, messages, max_tokens) + message, finish_reason, prompt_tokens = _query( + client, messages, max_tokens, use_tools=use_tools, enable_thinking=enable_thinking + ) break except _ContextOverflow as exc: # Prompt no longer fits: end without appending a turn so the @@ -766,7 +867,7 @@ def run_agent_loop( except _GatewayPaused as exc: if paused_for >= gateway_wait_s: log.warning("turn=%d gateway paused >%.0fs, giving up: %s", turn, gateway_wait_s, exc) - content, finish_reason, prompt_tokens = "", "error", 0 + message, finish_reason, prompt_tokens = {}, "error", 0 break log.info( "turn=%d gateway paused, waiting %.0fs (waited %.0fs): %s", turn, gateway_poll_s, paused_for, exc @@ -787,24 +888,36 @@ def run_agent_loop( if prompt_tokens > max_prompt_tokens: max_prompt_tokens = prompt_tokens - messages.append({"role": "assistant", "content": content}) + messages.append(message) try: - action = parse_action(content) - except FormatError as exc: + if use_tools: + action = parse_tool_action(message, finish_reason) + else: + action = parse_action( + message.get("content") or "", + enable_thinking=enable_thinking and "reasoning_content" not in message, + ) + except UnrenderableToolCall as exc: + # Do not rewrite the model's generated arguments or retry a history + # that vLLM cannot render; end with the recorded completion intact. + log.warning("turn=%d cannot replay tool call, ending episode: %s", turn, exc) + return False, turns_used, max_prompt_tokens + except (FormatError, ValueError) as exc: n_format_errors += 1 if 0 < max_format_errors <= n_format_errors: log.warning("turn=%d %d consecutive format errors, ending episode", turn, n_format_errors) return False, turns_used, max_prompt_tokens log.info("turn=%d format error (%d/%d)", turn, n_format_errors, max_format_errors) - messages.append({"role": "user", "content": format_error_message(exc.n_actions, finish_reason)}) + error = format_error_message(exc.n_actions, finish_reason) if isinstance(exc, FormatError) else str(exc) + _append_observation(messages, message, error) continue n_format_errors = 0 reason = _forbidden_action(action) if reason is not None: log.info("turn=%d blocked disallowed action: %r", turn, action[:200]) - messages.append({"role": "user", "content": render_observation(1, reason, obs_cap)}) + _append_observation(messages, message, render_observation(1, reason, obs_cap)) continue output, rc = _run(action, cmd_timeout) @@ -812,36 +925,56 @@ def run_agent_loop( if is_submission(output): log.info("turn=%d submit marker detected", turn) return True, turns_used, max_prompt_tokens - messages.append({"role": "user", "content": render_observation(rc, output, obs_cap)}) + _append_observation(messages, message, render_observation(rc, output, obs_cap)) return False, turns_used, max_prompt_tokens -def _query(client: Any, messages: list[dict[str, Any]], max_tokens: int) -> tuple[str, str, int]: - """Return ``(content, finish_reason, prompt_tokens)`` for one model call. +def _query( + client: Any, + messages: list[dict[str, Any]], + max_tokens: int, + *, + use_tools: bool = False, + enable_thinking: bool = False, +) -> tuple[dict[str, Any], str, int]: + """Return ``(assistant_message, finish_reason, prompt_tokens)`` for one model call. ``prompt_tokens`` is the server-reported ``usage.prompt_tokens`` (0 if the gateway omits usage), used for the prompt-length penalty — no local tokenizer. Raises ``_ContextOverflow`` when the prompt exceeds the model's context. - Any other failure yields ``("", "error", 0)`` so the caller can burn the turn. + Any other failure yields ``({}, "error", 0)`` so the caller can burn the turn. """ try: + tool_options = {"tools": [BASH_TOOL], "tool_choice": "auto", "parallel_tool_calls": False} if use_tools else {} completion = client.chat.completions.create( model="auto", messages=messages, max_tokens=max_tokens, temperature=1.0, - extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + extra_body={"chat_template_kwargs": {"enable_thinking": enable_thinking}}, + **tool_options, ) choice = completion.choices[0] prompt_tokens = getattr(getattr(completion, "usage", None), "prompt_tokens", 0) or 0 - return (choice.message.content or ""), (choice.finish_reason or "stop"), int(prompt_tokens) + message: dict[str, Any] = {"role": "assistant", "content": choice.message.content or ""} + if choice.message.tool_calls: + message["tool_calls"] = [call.model_dump(exclude_none=True) for call in choice.message.tool_calls] + # Older vLLM accepts reasoning on requests; Qwen's native template + # consumes reasoning_content. Keep both aliases for the round trip. + reasoning = getattr(choice.message, "reasoning", None) + if reasoning is None: + reasoning = getattr(choice.message, "reasoning_content", None) + if reasoning is not None: + message["reasoning"] = reasoning + message["reasoning_content"] = reasoning + return message, (choice.finish_reason or "stop"), int(prompt_tokens) except Exception as exc: if _is_context_overflow(exc): raise _ContextOverflow(str(exc)) from exc if _is_gateway_paused(exc): raise _GatewayPaused(str(exc)) from exc log.error("LLM call failed: %s", exc) - return "", "error", 0 + return {}, "error", 0 def main() -> int: @@ -868,6 +1001,8 @@ def main() -> int: max_tokens = int(os.environ.get("AGL_MAX_TOKENS", "16384")) max_format_errors = int(os.environ.get("SMITH_MAX_FORMAT_ERRORS", "3")) gateway_wait_s = float(os.environ.get("SMITH_GATEWAY_WAIT_S", "600")) + use_tools = os.environ.get("SMITH_USE_TOOLS", "0").lower() in ("1", "true", "yes") + enable_thinking = os.environ.get("SMITH_ENABLE_THINKING", "0").lower() in ("1", "true", "yes") f2p_only = os.environ.get("SMITH_F2P_ONLY", "1").lower() not in ("0", "false", "no") from openai import OpenAI @@ -886,6 +1021,8 @@ def main() -> int: max_tokens=max_tokens, max_format_errors=max_format_errors, gateway_wait_s=gateway_wait_s, + use_tools=use_tools, + enable_thinking=enable_thinking, ) patch = capture_patch() diff --git a/examples/swe_smith/job-template-openai.yaml b/examples/swe_smith/job-template-openai.yaml index f8ed419c5..cd0235879 100644 --- a/examples/swe_smith/job-template-openai.yaml +++ b/examples/swe_smith/job-template-openai.yaml @@ -23,6 +23,12 @@ spec: value: {{ input.problem_statement | yaml_escape }} - name: SMITH_MAX_TURNS value: "100" + # Opt in to native tool calls and thinking; see the Coding Agent docs + # for the matching vLLM parser and chat-template overrides. + - name: SMITH_USE_TOOLS + value: "0" + - name: SMITH_ENABLE_THINKING + value: "0" # Long-turn penalty (plan A): only SOLVED (reward==1) rollouts are # penalized for burning turns. reward = 1 - λ·clip((n_turns-T0)/(max_turns-T0),0,1). - name: SMITH_LEN_PEN_T0 diff --git a/tests/examples/test_swe_smith_tools.py b/tests/examples/test_swe_smith_tools.py new file mode 100644 index 000000000..e4d0980a4 --- /dev/null +++ b/tests/examples/test_swe_smith_tools.py @@ -0,0 +1,265 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Exercise the harness with the SDK messages returned by native tool parsers.""" + +from __future__ import annotations + +import copy +import json +from typing import Any + +import pytest +from openai.types.chat import ChatCompletion + +from examples.swe_smith.agents import smith_agent + + +def _call(command: str, *, call_id: str = "call_1", name: str = "execute_bash") -> dict[str, Any]: + return { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": json.dumps({"command": command})}, + } + + +class _Client: + def __init__(self, *messages: dict[str, Any], finish_reason: str = "tool_calls") -> None: + self.messages = iter(messages) + self.requests: list[dict[str, Any]] = [] + self.finish_reason = finish_reason + self.chat = self + self.completions = self + + def create(self, **kwargs: Any) -> ChatCompletion: + # Model the server's JSON decoding and Qwen template's mapping access, + # so a mocked completion cannot hide an invalid follow-up request. + for message in kwargs["messages"]: + for call in message.get("tool_calls", []): + arguments = json.loads(call["function"]["arguments"]) + assert isinstance(arguments, dict) + self.requests.append(copy.deepcopy(kwargs)) + return ChatCompletion.model_validate( + { + "id": "completion", + "created": 0, + "model": "Qwen/Qwen3.5-9B", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "finish_reason": self.finish_reason, + "message": {"role": "assistant", "content": None, **next(self.messages)}, + } + ], + "usage": {"prompt_tokens": 100 * len(self.requests), "completion_tokens": 10, "total_tokens": 110}, + } + ) + + +def _run_loop(client: _Client, **kwargs: Any) -> tuple[bool, int, int]: + return smith_agent.run_agent_loop( + client, "fix the bug", max_turns=2, cmd_timeout=1, obs_cap=1000, max_tokens=2048, **kwargs + ) + + +@pytest.mark.parametrize("reasoning_field", ["reasoning_content", "reasoning"]) +def test_native_calls_keep_reasoning_and_link_tool_results(monkeypatch, reasoning_field: str) -> None: + client = _Client( + {"tool_calls": [_call("ls -la")], reasoning_field: "Inspect the files first."}, + {"tool_calls": [_call(f"echo {smith_agent.SUBMIT_MARKER}", call_id="call_2")]}, + ) + commands: list[str] = [] + + def run(command: str, _timeout: int) -> tuple[str, int]: + commands.append(command) + return (smith_agent.SUBMIT_MARKER if command.startswith("echo ") else "file.py\n"), 0 + + monkeypatch.setattr(smith_agent, "_run", run) + assert _run_loop(client, use_tools=True, enable_thinking=True) == (True, 2, 200) + assert commands == ["ls -la", f"echo {smith_agent.SUBMIT_MARKER}"] + request = client.requests[1] + assert request["tools"] == [smith_agent.BASH_TOOL] + assert request["parallel_tool_calls"] is False + assert request["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True + assistant, observation = request["messages"][-2:] + assert assistant["tool_calls"] == [_call("ls -la")] + assert assistant["reasoning_content"] == "Inspect the files first." + assert assistant["reasoning"] == "Inspect the files first." + assert observation == { + "role": "tool", + "tool_call_id": "call_1", + "content": "0\n\nfile.py\n", + } + assert "bash code block" not in request["messages"][0]["content"] + assert "```bash" not in request["messages"][1]["content"] + + +@pytest.mark.parametrize( + "arguments", + [ + '{"command": 4}', + '{"command": " "}', + '{"other": "ls"}', + '{"command": "ls", "extra": true}', + ], +) +def test_bad_tool_arguments_are_reported_without_running(monkeypatch, arguments: str) -> None: + call = _call("unused") + call["function"]["arguments"] = arguments + client = _Client({"tool_calls": [call]}, {"tool_calls": [call]}) + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("malformed command ran")) + assert _run_loop(client, use_tools=True) == (False, 2, 200) + observation = client.requests[1]["messages"][-1] + assert observation["role"] == "tool" + assert observation["tool_call_id"] == "call_1" + assert "command" in observation["content"] + + +@pytest.mark.parametrize("arguments", ["{", "null", "[]", '"ls"']) +@pytest.mark.parametrize("variant", ["single", "parallel", "truncated", "unknown"]) +def test_unrenderable_tool_arguments_end_episode(monkeypatch, arguments: str, variant: str) -> None: + call = _call("unused", name="unknown" if variant == "unknown" else "execute_bash") + call["function"]["arguments"] = arguments + calls = [_call("ls", call_id="call_2"), call] if variant == "parallel" else [call] + client = _Client({"tool_calls": calls}, finish_reason="length" if variant == "truncated" else "tool_calls") + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("malformed command ran")) + assert _run_loop(client, use_tools=True) == (False, 1, 100) + assert len(client.requests) == 1 + + +@pytest.mark.parametrize( + "calls", + [[_call("ls", name="unknown")], [_call("ls"), _call("pwd", call_id="call_2")]], +) +def test_unknown_or_parallel_tools_reply_to_every_call(monkeypatch, calls: list[dict[str, Any]]) -> None: + client = _Client({"tool_calls": calls}, {"tool_calls": calls}) + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("invalid tool ran")) + assert _run_loop(client, use_tools=True) == (False, 2, 200) + observations = client.requests[1]["messages"][-len(calls) :] + assert [message["tool_call_id"] for message in observations] == [call["id"] for call in calls] + assert all(message["role"] == "tool" for message in observations) + + +@pytest.mark.parametrize("command", ["git show HEAD", "curl https://example.com", "pip install example"]) +def test_native_tools_keep_action_restrictions(monkeypatch, command: str) -> None: + client = _Client({"tool_calls": [_call(command)]}, {"tool_calls": [_call(command)]}) + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("forbidden command ran")) + assert _run_loop(client, use_tools=True) == (False, 2, 200) + observation = client.requests[1]["messages"][-1] + assert observation["role"] == "tool" + assert observation["tool_call_id"] == "call_1" + assert "1" in observation["content"] + + +def test_native_mode_does_not_execute_fenced_text(monkeypatch) -> None: + client = _Client({"content": "```bash\nls\n```"}, {"content": "```bash\nls\n```"}, finish_reason="stop") + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("non-tool command ran")) + assert _run_loop(client, use_tools=True) == (False, 2, 200) + assert client.requests[1]["messages"][-1] == { + "role": "user", + "content": "Expected exactly one execute_bash call, found 0.", + } + + +def test_truncated_native_call_is_not_executed(monkeypatch) -> None: + client = _Client({"tool_calls": [_call("ls")]}, {"tool_calls": [_call("ls")]}, finish_reason="length") + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("truncated command ran")) + assert _run_loop(client, use_tools=True) == (False, 2, 200) + assert "output token limit" in client.requests[1]["messages"][-1]["content"] + + +def test_default_bash_mode_remains_non_thinking(monkeypatch) -> None: + client = _Client({"content": "```bash\nls\n```"}, {"content": "```bash\npwd\n```"}, finish_reason="stop") + commands: list[str] = [] + monkeypatch.setattr(smith_agent, "_run", lambda command, _: (commands.append(command) or "ok", 0)) + assert _run_loop(client) == (False, 2, 200) + assert commands == ["ls", "pwd"] + assert "tools" not in client.requests[0] + assert client.requests[0]["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}} + assert client.requests[1]["messages"][-1]["role"] == "user" + + +@pytest.mark.parametrize("opening_tag", ["\n", ""]) +def test_bash_mode_ignores_commands_inside_raw_thinking(monkeypatch, opening_tag: str) -> None: + content = opening_tag + "Consider ```bash\nwrong\n```\n\n```bash\nls\n```" + client = _Client({"content": content}, {"content": content}, finish_reason="stop") + commands: list[str] = [] + monkeypatch.setattr(smith_agent, "_run", lambda command, _: (commands.append(command) or "ok", 0)) + assert _run_loop(client, enable_thinking=True) == (False, 2, 200) + assert commands == ["ls", "ls"] + assert client.requests[1]["messages"][-2]["content"] == content + + +def test_bash_mode_accepts_separate_reasoning(monkeypatch) -> None: + message = {"content": "```bash\nls\n```", "reasoning": "Consider ```bash\nwrong\n```"} + client = _Client(message, message, finish_reason="stop") + commands: list[str] = [] + monkeypatch.setattr(smith_agent, "_run", lambda command, _: (commands.append(command) or "ok", 0)) + assert _run_loop(client, enable_thinking=True) == (False, 2, 200) + assert commands == ["ls", "ls"] + assert client.requests[1]["messages"][-2]["reasoning_content"] == message["reasoning"] + + +@pytest.mark.parametrize("opening_tag", ["\n", ""]) +def test_unfinished_raw_thinking_does_not_execute_a_command(monkeypatch, opening_tag: str) -> None: + message = {"content": opening_tag + "Consider ```bash\nwrong\n```"} + client = _Client(message, message, finish_reason="length") + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("reasoning command ran")) + assert _run_loop(client, enable_thinking=True) == (False, 2, 200) + assert "output token limit" in client.requests[1]["messages"][-1]["content"] + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_main_reads_tool_and_thinking_settings(monkeypatch, enabled: bool) -> None: + monkeypatch.setenv("AGL_TASK_INPUT", "fix the bug") + monkeypatch.setenv("AGL_EVAL_META", '{"instance_id": "example"}') + monkeypatch.setenv("AGL_OPENAI_BASE_URL", "http://example.invalid/v1") + if enabled: + monkeypatch.setenv("SMITH_USE_TOOLS", "true") + monkeypatch.setenv("SMITH_ENABLE_THINKING", "1") + else: + monkeypatch.delenv("SMITH_USE_TOOLS", raising=False) + monkeypatch.delenv("SMITH_ENABLE_THINKING", raising=False) + settings: dict[str, Any] = {} + + def run_loop(*_args: Any, **kwargs: Any) -> tuple[bool, int, int]: + settings.update(kwargs) + return False, 0, 0 + + monkeypatch.setattr("openai.OpenAI", lambda **_kwargs: object()) + monkeypatch.setattr(smith_agent, "checkout_bug_commit", lambda *_args: None) + monkeypatch.setattr(smith_agent, "relocate_git", lambda: None) + monkeypatch.setattr(smith_agent, "run_agent_loop", run_loop) + monkeypatch.setattr(smith_agent, "capture_patch", lambda: "") + monkeypatch.setattr(smith_agent, "evaluate", lambda *_args, **_kwargs: (0.0, False, "test", False)) + monkeypatch.setattr(smith_agent, "post_event", lambda *_args, **_kwargs: None) + assert smith_agent.main() == 0 + assert settings["use_tools"] is enabled + assert settings["enable_thinking"] is enabled + + +def test_tool_format_error_can_recover_and_submit(monkeypatch) -> None: + client = _Client( + {"tool_calls": [_call("ls", name="unknown")]}, + {"tool_calls": [_call(f"echo {smith_agent.SUBMIT_MARKER}", call_id="call_2")]}, + ) + commands: list[str] = [] + monkeypatch.setattr( + smith_agent, "_run", lambda command, _: (commands.append(command) or smith_agent.SUBMIT_MARKER, 0) + ) + assert _run_loop(client, use_tools=True) == (True, 2, 200) + assert commands == [f"echo {smith_agent.SUBMIT_MARKER}"] + + +def test_native_format_errors_respect_the_error_limit(monkeypatch) -> None: + client = _Client({"tool_calls": [_call("ls", name="unknown")]}) + monkeypatch.setattr(smith_agent, "_run", lambda *_args: pytest.fail("invalid tool ran")) + assert _run_loop(client, use_tools=True, max_format_errors=1) == (False, 1, 100) + assert len(client.requests) == 1 + + +def test_canonical_reasoning_takes_precedence_over_empty_legacy_field() -> None: + client = _Client({"reasoning": "Inspect the files.", "reasoning_content": "", "tool_calls": [_call("ls")]}) + message, _, _ = smith_agent._query(client, [], 2048, use_tools=True, enable_thinking=True) + assert message["reasoning"] == "Inspect the files." + assert message["reasoning_content"] == "Inspect the files."