From 99da1184f07000323ee85979404dbedf5eb166ee Mon Sep 17 00:00:00 2001 From: DennyHo0917 <149746199+DennyHo0917@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:36:10 +0800 Subject: [PATCH 1/2] feat(eval): add optional API Route reader Signed-off-by: DennyHo0917 <149746199+DennyHo0917@users.noreply.github.com> --- eval/README.md | 27 ++++++++ eval/lib/model_config.py | 13 ++++ eval/run_bench.py | 28 ++++++++- tests/test_api_route_client.py | 112 +++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 tests/test_api_route_client.py diff --git a/eval/README.md b/eval/README.md index dcf1d08..570f143 100644 --- a/eval/README.md +++ b/eval/README.md @@ -234,6 +234,33 @@ Google SDK. `--api-key` overrides `CHEAPER_INFERENCE_API_KEY`, and other provide environment keys are never used. Do not combine this option with another provider flag or `--api-base`. +## API Route readers + +[API Route](https://api-route.com) provides an OpenAI-compatible endpoint. +Select it explicitly with `--api-route` and an exact chat-completions model ID +available to your account (listed by authenticated `GET +https://global.api-route.com/v1/models`). Existing reader defaults are unchanged. + +```bash +export API_ROUTE_API_KEY=your_api_route_key +export API_ROUTE_MODEL=your_chat_completions_model_id +python run_bench.py --task simpleqa --model "$API_ROUTE_MODEL" \ + --api-route --num-examples 1 --max-concurrent 1 --max-tokens 256 +``` + +Set `API_ROUTE_MODEL` to your chosen model ID before running this text-only, +no-retrieval example. For screenshot benchmarks, select a model whose image +input support you have verified; a model-list entry alone does not establish +that support. Model IDs are forwarded unchanged to +`https://global.api-route.com/v1` through the OpenAI-compatible client, including +IDs containing `gemini` (which do not select the native Google SDK). + +`--api-key` overrides `API_ROUTE_API_KEY`; other providers' environment keys are +never used. Do not combine `--api-route` with another provider flag, `--litellm`, +or `--api-base`. API Route requests are not automatically retried, including +timeouts, connection errors and rate limits, to avoid duplicating billable +generation requests. + ## MiniMax readers Set `MINIMAX_API_KEY` and select either registered model ID. The model context length diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py index 0b89844..6979ba5 100644 --- a/eval/lib/model_config.py +++ b/eval/lib/model_config.py @@ -85,6 +85,19 @@ def get_cheaperinference_config( } +def get_api_route_config(model_name: str, api_key: str | None = None) -> Dict[str, Any]: + """Resolve an explicitly selected API Route reader with its own credentials.""" + if not api_key or api_key == "dummy": + api_key = os.getenv("API_ROUTE_API_KEY") + if not api_key or not api_key.strip() or api_key == "dummy": + raise ValueError("API Route requires --api-key or API_ROUTE_API_KEY.") + return { + "api_base": "https://global.api-route.com/v1", + "api_key": api_key, + "model": model_name, + } + + def get_model_config(model_name: str) -> Dict[str, Any]: """ Get model configuration based on model name. diff --git a/eval/run_bench.py b/eval/run_bench.py index ec3dd25..e90eaa0 100644 --- a/eval/run_bench.py +++ b/eval/run_bench.py @@ -73,6 +73,7 @@ ) from lib.model_config import ( ORCAROUTER_API_BASE, + get_api_route_config, get_atlascloud_config, get_cheaperinference_config, get_model_config, @@ -963,6 +964,8 @@ async def run_async(args): model_config = get_atlascloud_config(args.model, args.api_key) elif args.cheaperinference: model_config = get_cheaperinference_config(args.model, args.api_key) + elif args.api_route: + model_config = get_api_route_config(args.model, args.api_key) else: model_config = get_model_config(args.model) @@ -977,6 +980,11 @@ async def run_async(args): api_key = model_config["api_key"] model = model_config["model"] logger.info(f"Using Cheaper Inference API with model: {model}") + elif args.api_route: + api_base = model_config["api_base"] + api_key = model_config["api_key"] + model = model_config["model"] + logger.info(f"Using API Route API with model: {model}") elif args.open_router: api_base = "https://openrouter.ai/api/v1" if args.api_key and args.api_key != "dummy": @@ -1162,9 +1170,10 @@ async def run_async(args): or args.orcarouter or args.atlascloud or args.cheaperinference + or args.api_route ), use_litellm=args.litellm, - retry_requests=not args.atlascloud, + retry_requests=not (args.atlascloud or args.api_route), ) # 3b. Create pixel-compressed encoder for generation if requested @@ -1404,6 +1413,11 @@ def main(): action="store_true", help="Use Cheaper Inference (https://api.cheaperinference.com/v1) with a bare model ID. Requires --api-key or CHEAPER_INFERENCE_API_KEY.", ) + parser.add_argument( + "--api-route", + action="store_true", + help="Use API Route with an exact catalog model ID. Requires --api-key or API_ROUTE_API_KEY.", + ) parser.add_argument( "--litellm", action="store_true", @@ -1973,6 +1987,18 @@ def main(): if args.cheaperinference and args.api_base: parser.error("--cheaperinference uses its fixed endpoint; omit --api-base") + if args.api_route and ( + args.open_router + or args.commonstack + or args.orcarouter + or args.atlascloud + or args.cheaperinference + or args.litellm + ): + parser.error("--api-route cannot be combined with another provider flag") + if args.api_route and args.api_base: + parser.error("--api-route uses its fixed endpoint; omit --api-base") + # Validate mutually exclusive options mode_count = sum( [ diff --git a/tests/test_api_route_client.py b/tests/test_api_route_client.py new file mode 100644 index 0000000..62f1f82 --- /dev/null +++ b/tests/test_api_route_client.py @@ -0,0 +1,112 @@ +"""Offline coverage of the optional API Route evaluation reader.""" + +import asyncio +import sys +import types +from pathlib import Path +from runpy import run_path + +import pytest + +_LIB = Path(__file__).parents[1] / "eval" / "lib" +if "lib" not in sys.modules: + _pkg = types.ModuleType("lib") + _pkg.__path__ = [str(_LIB)] + sys.modules["lib"] = _pkg + +from lib.llm import LLMClient + +get_config = run_path(str(_LIB / "model_config.py"))["get_api_route_config"] + + +@pytest.mark.parametrize("model", ["gpt-5.5", "google/gemini-fixture"]) +def test_api_route_preserves_model_and_isolates_credentials(monkeypatch, model): + monkeypatch.setenv("API_ROUTE_API_KEY", "api-route-test-key") + monkeypatch.setenv("OPENAI_API_KEY", "other-key") + monkeypatch.setenv("API_KEY", "generic-key") + config = get_config(model) + assert config == { + "model": model, + "api_base": "https://global.api-route.com/v1", + "api_key": "api-route-test-key", + } + assert get_config(model, "explicit-key")["api_key"] == "explicit-key" + + +@pytest.mark.parametrize("key", [None, "", " ", "dummy"]) +def test_api_route_requires_own_key(monkeypatch, key): + monkeypatch.delenv("API_ROUTE_API_KEY", raising=False) + if key is not None: + monkeypatch.setenv("API_ROUTE_API_KEY", key) + monkeypatch.setenv("API_KEY", "generic-key") + monkeypatch.setenv("GOOGLE_API_KEY", "google-key") + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + with pytest.raises(ValueError, match="API_ROUTE_API_KEY"): + get_config("deepseek-ai/deepseek-v3.2") + + +@pytest.mark.parametrize( + "error", + [ + None, + asyncio.TimeoutError(), + RuntimeError("Connection error"), + RuntimeError("429"), + ], +) +def test_api_route_client_routes_once(monkeypatch, error): + calls = [] + options = {} + + async def create(**kwargs): + calls.append(kwargs) + if error is not None: + raise error + return types.SimpleNamespace( + choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))], + usage=types.SimpleNamespace( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def openai_client(**kwargs): + options.update(kwargs) + return types.SimpleNamespace( + chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=create)) + ) + + monkeypatch.setitem( + sys.modules, "openai", types.SimpleNamespace(AsyncOpenAI=openai_client) + ) + client = LLMClient( + **get_config("google/gemini-fixture", "api-route-test-key"), + force_openai_compat=True, + retry_requests=False, + max_tokens=64, + ) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + }, + {"type": "text", "text": "2+2?"}, + ], + } + ] + if error is None: + text, usage = asyncio.run(client.generate(messages)) + assert text == "4" + assert usage["total_tokens"] == 4 + else: + with pytest.raises(type(error)): + asyncio.run(client.generate(messages)) + assert len(calls) == 1 + assert calls[0]["model"] == "google/gemini-fixture" + assert calls[0]["messages"] == messages + assert options["base_url"] == "https://global.api-route.com/v1" + assert options["api_key"] == "api-route-test-key" + assert options["max_retries"] == 0 + assert not client.is_gemini From 9ff3427aa9f7770e3de068802dcafbd854cc62fb Mon Sep 17 00:00:00 2001 From: DennyHo0917 <149746199+DennyHo0917@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:50:56 +0800 Subject: [PATCH 2/2] fix(eval): keep API Route on the shared retry policy --- eval/README.md | 8 +++-- eval/run_bench.py | 3 +- tests/test_api_route_client.py | 64 +++++++++++++++++++++++++++------- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/eval/README.md b/eval/README.md index 570f143..2d030ac 100644 --- a/eval/README.md +++ b/eval/README.md @@ -257,9 +257,11 @@ IDs containing `gemini` (which do not select the native Google SDK). `--api-key` overrides `API_ROUTE_API_KEY`; other providers' environment keys are never used. Do not combine `--api-route` with another provider flag, `--litellm`, -or `--api-base`. API Route requests are not automatically retried, including -timeouts, connection errors and rate limits, to avoid duplicating billable -generation requests. +or `--api-base`. API Route uses the shared `LLMClient` retry policy for timeouts, +connection errors and rate limits. SDK-level retries remain disabled so retries +are controlled by the benchmark client rather than stacked across both layers. +This is client-side behavior, not a guarantee of provider-side idempotency or +billing behavior. ## MiniMax readers diff --git a/eval/run_bench.py b/eval/run_bench.py index e90eaa0..a1a61f5 100644 --- a/eval/run_bench.py +++ b/eval/run_bench.py @@ -1173,7 +1173,8 @@ async def run_async(args): or args.api_route ), use_litellm=args.litellm, - retry_requests=not (args.atlascloud or args.api_route), + # API Route uses the shared retry policy; preserve Atlas Cloud's existing opt-out. + retry_requests=not args.atlascloud, ) # 3b. Create pixel-compressed encoder for generation if requested diff --git a/tests/test_api_route_client.py b/tests/test_api_route_client.py index 62f1f82..37a126d 100644 --- a/tests/test_api_route_client.py +++ b/tests/test_api_route_client.py @@ -1,10 +1,12 @@ """Offline coverage of the optional API Route evaluation reader.""" +import ast import asyncio import sys import types from pathlib import Path from runpy import run_path +from unittest.mock import AsyncMock import pytest @@ -46,21 +48,35 @@ def test_api_route_requires_own_key(monkeypatch, key): @pytest.mark.parametrize( - "error", + ("error", "failures", "expected_calls"), [ - None, - asyncio.TimeoutError(), - RuntimeError("Connection error"), - RuntimeError("429"), + pytest.param(None, 0, 1, id="success"), + pytest.param(asyncio.TimeoutError(), 1, 2, id="timeout-recovery"), + pytest.param( + RuntimeError("Request timed out"), 1, 2, id="sdk-timeout-recovery" + ), + pytest.param(RuntimeError("Connection error"), 1, 2, id="connection-recovery"), + pytest.param(RuntimeError("429"), 1, 2, id="rate-limit-recovery"), + pytest.param(asyncio.TimeoutError(), 3, 3, id="timeout-limit"), + pytest.param(RuntimeError("Connection error"), 12, 12, id="connection-limit"), + pytest.param(RuntimeError("429"), 6, 6, id="rate-limit-limit"), + pytest.param(RuntimeError("400 bad request"), 1, 1, id="nonretryable-error"), + pytest.param( + RuntimeError("500 internal server error"), 1, 1, id="no-sdk-retry" + ), ], ) -def test_api_route_client_routes_once(monkeypatch, error): +def test_api_route_client_uses_shared_retries( + monkeypatch, error, failures, expected_calls +): calls = [] options = {} + sleep = AsyncMock() + monkeypatch.setattr(asyncio, "sleep", sleep) async def create(**kwargs): calls.append(kwargs) - if error is not None: + if error is not None and len(calls) <= failures: raise error return types.SimpleNamespace( choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))], @@ -81,7 +97,6 @@ def openai_client(**kwargs): client = LLMClient( **get_config("google/gemini-fixture", "api-route-test-key"), force_openai_compat=True, - retry_requests=False, max_tokens=64, ) messages = [ @@ -96,17 +111,42 @@ def openai_client(**kwargs): ], } ] - if error is None: + if failures < expected_calls: text, usage = asyncio.run(client.generate(messages)) assert text == "4" assert usage["total_tokens"] == 4 else: with pytest.raises(type(error)): asyncio.run(client.generate(messages)) - assert len(calls) == 1 - assert calls[0]["model"] == "google/gemini-fixture" - assert calls[0]["messages"] == messages + assert len(calls) == expected_calls + assert sleep.await_count == expected_calls - 1 + assert all(call["model"] == "google/gemini-fixture" for call in calls) + assert all(call["messages"] == messages for call in calls) assert options["base_url"] == "https://global.api-route.com/v1" assert options["api_key"] == "api-route-test-key" assert options["max_retries"] == 0 + assert client.retry_requests assert not client.is_gemini + + +@pytest.mark.parametrize( + ("api_route", "atlascloud", "expected"), + [(False, False, True), (True, False, True), (False, True, False)], +) +def test_cli_retry_policy(api_route, atlascloud, expected): + # Check the actual CLI wiring without importing the dataset/benchmark stack. + source = (_LIB.parent / "run_bench.py").read_text() + calls = [ + node + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "LLMClient" + ] + assert len(calls) == 1 + retry = next(kw.value for kw in calls[0].keywords if kw.arg == "retry_requests") + args = types.SimpleNamespace(api_route=api_route, atlascloud=atlascloud) + assert ( + eval(compile(ast.Expression(retry), "run_bench.py", "eval"), {"args": args}) + is expected + )