diff --git a/browseruse_bench/agents/browser_use.py b/browseruse_bench/agents/browser_use.py index e879efd..40197ec 100644 --- a/browseruse_bench/agents/browser_use.py +++ b/browseruse_bench/agents/browser_use.py @@ -223,28 +223,64 @@ def patched(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]: setattr(SchemaOptimizer, _PATCHED_SCHEMA_OPTIMIZER_ATTR, True) -def _enable_claude_thinking(llm: Any, reasoning_effort: str) -> None: - """Inject Claude reasoning params for OpenAI-compatible gateways.""" +def _normalize_allowed_openai_params(value: Any) -> list[str]: + """Validate and de-duplicate LiteLLM's per-request parameter allowlist.""" + if value is None: + return [] + if not isinstance(value, (list, tuple)) or not all( + isinstance(param, str) and param for param in value + ): + raise ValueError("allowed_openai_params must be a list of non-empty strings") + return list(dict.fromkeys(value)) + + +def _enable_litellm_chat_passthrough( + llm: Any, + allowed_openai_params: list[str], + *, + extra_body_defaults: dict[str, Any] | None = None, +) -> None: + """Inject LiteLLM-only fields into each Chat Completions request. + + browser-use's pinned ``ChatOpenAI`` does not expose OpenAI SDK + ``extra_body`` as a constructor option. LiteLLM 1.93 requires + ``allowed_openai_params`` in the request body when its capability metadata + does not yet recognize a parameter that the upstream endpoint accepts, so + wrap the fresh client returned for every invocation. + """ + injected_allowed = _normalize_allowed_openai_params(allowed_openai_params) + defaults = dict(extra_body_defaults or {}) original_get_client = llm.get_client - def get_client_with_thinking() -> Any: + def get_client_with_passthrough() -> Any: client = original_get_client() original_create = client.chat.completions.create - async def create_with_thinking(*args: Any, **kwargs: Any) -> Any: + async def create_with_passthrough(*args: Any, **kwargs: Any) -> Any: extra_body = dict(kwargs.get("extra_body") or {}) - extra_body.setdefault("reasoning_effort", reasoning_effort) - allowed = list(extra_body.get("allowed_openai_params") or []) - if "reasoning_effort" not in allowed: - allowed.append("reasoning_effort") + for key, value in defaults.items(): + extra_body.setdefault(key, value) + allowed = _normalize_allowed_openai_params(extra_body.get("allowed_openai_params")) + for param in injected_allowed: + if param not in allowed: + allowed.append(param) extra_body["allowed_openai_params"] = allowed kwargs["extra_body"] = extra_body return await original_create(*args, **kwargs) - client.chat.completions.create = create_with_thinking # type: ignore[method-assign] + client.chat.completions.create = create_with_passthrough # type: ignore[method-assign] return client - llm.get_client = get_client_with_thinking # type: ignore[method-assign] + llm.get_client = get_client_with_passthrough # type: ignore[method-assign] + + +def _enable_claude_thinking(llm: Any, reasoning_effort: str) -> None: + """Inject Claude reasoning params for OpenAI-compatible gateways.""" + _enable_litellm_chat_passthrough( + llm, + ["reasoning_effort"], + extra_body_defaults={"reasoning_effort": reasoning_effort}, + ) class _LLMFailureRecorder: @@ -970,6 +1006,21 @@ def _create_llm( kwargs = provider_builders[model_type](model_id, agent_config, config_info) llm = llm_class(**kwargs) + allowed_openai_params = _normalize_allowed_openai_params( + agent_config.get("allowed_openai_params") + ) + if allowed_openai_params: + if model_type != "OPENAI": + raise ValueError( + "allowed_openai_params is only supported for OPENAI Chat Completions models" + ) + if agent_config.get("model_api_style") == "responses": + raise ValueError( + "allowed_openai_params passthrough is only supported for Chat Completions" + ) + _enable_litellm_chat_passthrough(llm, allowed_openai_params) + config_info["allowed_openai_params"] = allowed_openai_params + if is_claude and model_type in ("OPENAI", "AZURE"): thinking_enabled = _get_config_value( agent_config, diff --git a/config.example.yaml b/config.example.yaml index edfca03..efab84b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -36,6 +36,10 @@ models: base_url: $OPENAI_BASE_URL max_tokens: 16384 reasoning_effort: xhigh + # LiteLLM 1.93 rejects reasoning_effort for this custom model alias before + # forwarding it; explicitly allow the parameter for this request. + allowed_openai_params: + - reasoning_effort llm_timeout: 150 dont_force_structured_output: false add_schema_to_system_prompt: true diff --git a/tests/browseruse_bench/test_browser_use_agent.py b/tests/browseruse_bench/test_browser_use_agent.py index 2ce641f..f1775f9 100644 --- a/tests/browseruse_bench/test_browser_use_agent.py +++ b/tests/browseruse_bench/test_browser_use_agent.py @@ -204,6 +204,124 @@ def test_build_openai_kwargs_rejects_invalid_reasoning_effort() -> None: ) +def test_litellm_chat_passthrough_merges_allowed_openai_params() -> None: + captured: dict[str, Any] = {} + + class FakeCompletions: + async def create(self, *args: Any, **kwargs: Any) -> str: + captured["kwargs"] = kwargs + return "ok" + + class FakeLLM: + def get_client(self) -> Any: + return SimpleNamespace(chat=SimpleNamespace(completions=FakeCompletions())) + + llm = FakeLLM() + browser_use_module._enable_litellm_chat_passthrough( + llm, + ["reasoning_effort", "reasoning_effort"], + ) + + result = asyncio.run( + llm.get_client().chat.completions.create( + messages=[], + extra_body={ + "allowed_openai_params": ["tools"], + "gateway_trace": "keep-me", + }, + ) + ) + + assert result == "ok" + assert captured["kwargs"]["extra_body"] == { + "allowed_openai_params": ["tools", "reasoning_effort"], + "gateway_trace": "keep-me", + } + + +def test_create_llm_enables_configured_litellm_passthrough( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + class FakeLLM: + def __init__(self, **kwargs: Any) -> None: + captured["kwargs"] = kwargs + + def fake_enable_passthrough( + llm: Any, + allowed_openai_params: list[str], + *, + extra_body_defaults: dict[str, Any] | None = None, + ) -> None: + captured["passthrough_llm"] = llm + captured["allowed_openai_params"] = allowed_openai_params + captured["extra_body_defaults"] = extra_body_defaults + + monkeypatch.setattr(browser_use_module, "ChatOpenAI", FakeLLM) + monkeypatch.setattr( + browser_use_module, + "_enable_litellm_chat_passthrough", + fake_enable_passthrough, + ) + + config_info: dict[str, Any] = {} + llm = BrowserUseAgent()._create_llm( + "OPENAI", + "gpt-5.6-sol", + { + "api_key": "key", + "base_url": "https://gateway.example/v1", + "reasoning_effort": "xhigh", + "allowed_openai_params": [ + "reasoning_effort", + "reasoning_effort", + ], + }, + config_info, + ) + + assert captured["passthrough_llm"] is llm + assert captured["allowed_openai_params"] == ["reasoning_effort"] + assert config_info["allowed_openai_params"] == ["reasoning_effort"] + + +def test_create_llm_rejects_invalid_allowed_openai_params() -> None: + with pytest.raises(ValueError, match="allowed_openai_params"): + BrowserUseAgent()._create_llm( + "OPENAI", + "gpt-5.6-sol", + { + "api_key": "key", + "base_url": "https://gateway.example/v1", + "allowed_openai_params": "reasoning_effort", + }, + {}, + ) + + +def test_create_llm_rejects_passthrough_for_azure_model_type( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeLLM: + def __init__(self, **kwargs: Any) -> None: + pass + + monkeypatch.setattr(browser_use_module, "ChatAzureOpenAI", FakeLLM) + + with pytest.raises(ValueError, match="OPENAI Chat Completions"): + BrowserUseAgent()._create_llm( + "AZURE", + "gpt-5.6-sol", + { + "api_key": "key", + "base_url": "https://gateway.example/v1", + "allowed_openai_params": ["reasoning_effort"], + }, + {}, + ) + + class _OutputForParserTest(BaseModel): memory: str action: list[dict[str, Any]]