feat(eval): add optional API Route reader

eval/README.md

@@ -234,6 +234,35 @@ Google SDK. `--api-key` overrides `CHEAPER_INFERENCE_API_KEY`, and other provide
 environment keys are never used. Do not combine this option with another provider
 flag or `--api-base`.
 
+## API Route readers
+
+[API Route](https://api-route.com) provides an OpenAI-compatible endpoint.
+Select it explicitly with `--api-route` and an exact chat-completions model ID
+available to your account (listed by authenticated `GET
+https://global.api-route.com/v1/models`). Existing reader defaults are unchanged.
+
+```bash
+export API_ROUTE_API_KEY=your_api_route_key
+export API_ROUTE_MODEL=your_chat_completions_model_id
+python run_bench.py --task simpleqa --model "$API_ROUTE_MODEL" \
+    --api-route --num-examples 1 --max-concurrent 1 --max-tokens 256
+```
+
+Set `API_ROUTE_MODEL` to your chosen model ID before running this text-only,
+no-retrieval example. For screenshot benchmarks, select a model whose image
+input support you have verified; a model-list entry alone does not establish
+that support. Model IDs are forwarded unchanged to
+`https://global.api-route.com/v1` through the OpenAI-compatible client, including
+IDs containing `gemini` (which do not select the native Google SDK).
+
+`--api-key` overrides `API_ROUTE_API_KEY`; other providers' environment keys are
+never used. Do not combine `--api-route` with another provider flag, `--litellm`,
+or `--api-base`. API Route uses the shared `LLMClient` retry policy for timeouts,
+connection errors and rate limits. SDK-level retries remain disabled so retries
+are controlled by the benchmark client rather than stacked across both layers.
+This is client-side behavior, not a guarantee of provider-side idempotency or
+billing behavior.
+
 ## MiniMax readers
 
 Set `MINIMAX_API_KEY` and select either registered model ID. The model context length

eval/lib/model_config.py

@@ -85,6 +85,19 @@ def get_cheaperinference_config(
     }
 
 
+def get_api_route_config(model_name: str, api_key: str | None = None) -> Dict[str, Any]:
+    """Resolve an explicitly selected API Route reader with its own credentials."""
+    if not api_key or api_key == "dummy":
+        api_key = os.getenv("API_ROUTE_API_KEY")
+    if not api_key or not api_key.strip() or api_key == "dummy":
+        raise ValueError("API Route requires --api-key or API_ROUTE_API_KEY.")
+    return {
+        "api_base": "https://global.api-route.com/v1",
+        "api_key": api_key,
+        "model": model_name,
+    }
+
+
 def get_model_config(model_name: str) -> Dict[str, Any]:
     """
     Get model configuration based on model name.

eval/run_bench.py

@@ -73,6 +73,7 @@
 )
 from lib.model_config import (
     ORCAROUTER_API_BASE,
+    get_api_route_config,
     get_atlascloud_config,
     get_cheaperinference_config,
     get_model_config,
@@ -963,6 +964,8 @@ async def run_async(args):
         model_config = get_atlascloud_config(args.model, args.api_key)
     elif args.cheaperinference:
         model_config = get_cheaperinference_config(args.model, args.api_key)
+    elif args.api_route:
+        model_config = get_api_route_config(args.model, args.api_key)
     else:
         model_config = get_model_config(args.model)
 
@@ -977,6 +980,11 @@ async def run_async(args):
         api_key = model_config["api_key"]
         model = model_config["model"]
         logger.info(f"Using Cheaper Inference API with model: {model}")
+    elif args.api_route:
+        api_base = model_config["api_base"]
+        api_key = model_config["api_key"]
+        model = model_config["model"]
+        logger.info(f"Using API Route API with model: {model}")
     elif args.open_router:
         api_base = "https://openrouter.ai/api/v1"
         if args.api_key and args.api_key != "dummy":
@@ -1162,8 +1170,10 @@ async def run_async(args):
             or args.orcarouter
             or args.atlascloud
             or args.cheaperinference
+            or args.api_route
         ),
         use_litellm=args.litellm,
+        # API Route uses the shared retry policy; preserve Atlas Cloud's existing opt-out.
         retry_requests=not args.atlascloud,
     )
 
@@ -1404,6 +1414,11 @@ def main():
         action="store_true",
         help="Use Cheaper Inference (https://api.cheaperinference.com/v1) with a bare model ID. Requires --api-key or CHEAPER_INFERENCE_API_KEY.",
     )
+    parser.add_argument(
+        "--api-route",
+        action="store_true",
+        help="Use API Route with an exact catalog model ID. Requires --api-key or API_ROUTE_API_KEY.",
+    )
     parser.add_argument(
         "--litellm",
         action="store_true",
@@ -1973,6 +1988,18 @@ def main():
     if args.cheaperinference and args.api_base:
         parser.error("--cheaperinference uses its fixed endpoint; omit --api-base")
 
+    if args.api_route and (
+        args.open_router
+        or args.commonstack
+        or args.orcarouter
+        or args.atlascloud
+        or args.cheaperinference
+        or args.litellm
+    ):
+        parser.error("--api-route cannot be combined with another provider flag")
+    if args.api_route and args.api_base:
+        parser.error("--api-route uses its fixed endpoint; omit --api-base")
+
     # Validate mutually exclusive options
     mode_count = sum(
         [

tests/test_api_route_client.py

@@ -0,0 +1,152 @@
+"""Offline coverage of the optional API Route evaluation reader."""
+
+import ast
+import asyncio
+import sys
+import types
+from pathlib import Path
+from runpy import run_path
+from unittest.mock import AsyncMock
+
+import pytest
+
+_LIB = Path(__file__).parents[1] / "eval" / "lib"
+if "lib" not in sys.modules:
+    _pkg = types.ModuleType("lib")
+    _pkg.__path__ = [str(_LIB)]
+    sys.modules["lib"] = _pkg
+
+from lib.llm import LLMClient
+
+get_config = run_path(str(_LIB / "model_config.py"))["get_api_route_config"]
+
+
+@pytest.mark.parametrize("model", ["gpt-5.5", "google/gemini-fixture"])
+def test_api_route_preserves_model_and_isolates_credentials(monkeypatch, model):
+    monkeypatch.setenv("API_ROUTE_API_KEY", "api-route-test-key")
+    monkeypatch.setenv("OPENAI_API_KEY", "other-key")
+    monkeypatch.setenv("API_KEY", "generic-key")
+    config = get_config(model)
+    assert config == {
+        "model": model,
+        "api_base": "https://global.api-route.com/v1",
+        "api_key": "api-route-test-key",
+    }
+    assert get_config(model, "explicit-key")["api_key"] == "explicit-key"
+
+
+@pytest.mark.parametrize("key", [None, "", " ", "dummy"])
+def test_api_route_requires_own_key(monkeypatch, key):
+    monkeypatch.delenv("API_ROUTE_API_KEY", raising=False)
+    if key is not None:
+        monkeypatch.setenv("API_ROUTE_API_KEY", key)
+    monkeypatch.setenv("API_KEY", "generic-key")
+    monkeypatch.setenv("GOOGLE_API_KEY", "google-key")
+    monkeypatch.setenv("OPENAI_API_KEY", "openai-key")
+    with pytest.raises(ValueError, match="API_ROUTE_API_KEY"):
+        get_config("deepseek-ai/deepseek-v3.2")
+
+
+@pytest.mark.parametrize(
+    ("error", "failures", "expected_calls"),
+    [
+        pytest.param(None, 0, 1, id="success"),
+        pytest.param(asyncio.TimeoutError(), 1, 2, id="timeout-recovery"),
+        pytest.param(
+            RuntimeError("Request timed out"), 1, 2, id="sdk-timeout-recovery"
+        ),
+        pytest.param(RuntimeError("Connection error"), 1, 2, id="connection-recovery"),
+        pytest.param(RuntimeError("429"), 1, 2, id="rate-limit-recovery"),
+        pytest.param(asyncio.TimeoutError(), 3, 3, id="timeout-limit"),
+        pytest.param(RuntimeError("Connection error"), 12, 12, id="connection-limit"),
+        pytest.param(RuntimeError("429"), 6, 6, id="rate-limit-limit"),
+        pytest.param(RuntimeError("400 bad request"), 1, 1, id="nonretryable-error"),
+        pytest.param(
+            RuntimeError("500 internal server error"), 1, 1, id="no-sdk-retry"
+        ),
+    ],
+)
+def test_api_route_client_uses_shared_retries(
+    monkeypatch, error, failures, expected_calls
+):
+    calls = []
+    options = {}
+    sleep = AsyncMock()
+    monkeypatch.setattr(asyncio, "sleep", sleep)
+
+    async def create(**kwargs):
+        calls.append(kwargs)
+        if error is not None and len(calls) <= failures:
+            raise error
+        return types.SimpleNamespace(
+            choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))],
+            usage=types.SimpleNamespace(
+                prompt_tokens=3, completion_tokens=1, total_tokens=4
+            ),
+        )
+
+    def openai_client(**kwargs):
+        options.update(kwargs)
+        return types.SimpleNamespace(
+            chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=create))
+        )
+
+    monkeypatch.setitem(
+        sys.modules, "openai", types.SimpleNamespace(AsyncOpenAI=openai_client)
+    )
+    client = LLMClient(
+        **get_config("google/gemini-fixture", "api-route-test-key"),
+        force_openai_compat=True,
+        max_tokens=64,
+    )
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,AAAA"},
+                },
+                {"type": "text", "text": "2+2?"},
+            ],
+        }
+    ]
+    if failures < expected_calls:
+        text, usage = asyncio.run(client.generate(messages))
+        assert text == "4"
+        assert usage["total_tokens"] == 4
+    else:
+        with pytest.raises(type(error)):
+            asyncio.run(client.generate(messages))
+    assert len(calls) == expected_calls
+    assert sleep.await_count == expected_calls - 1
+    assert all(call["model"] == "google/gemini-fixture" for call in calls)
+    assert all(call["messages"] == messages for call in calls)
+    assert options["base_url"] == "https://global.api-route.com/v1"
+    assert options["api_key"] == "api-route-test-key"
+    assert options["max_retries"] == 0
+    assert client.retry_requests
+    assert not client.is_gemini
+
+
+@pytest.mark.parametrize(
+    ("api_route", "atlascloud", "expected"),
+    [(False, False, True), (True, False, True), (False, True, False)],
+)
+def test_cli_retry_policy(api_route, atlascloud, expected):
+    # Check the actual CLI wiring without importing the dataset/benchmark stack.
+    source = (_LIB.parent / "run_bench.py").read_text()
+    calls = [
+        node
+        for node in ast.walk(ast.parse(source))
+        if isinstance(node, ast.Call)
+        and isinstance(node.func, ast.Name)
+        and node.func.id == "LLMClient"
+    ]
+    assert len(calls) == 1
+    retry = next(kw.value for kw in calls[0].keywords if kw.arg == "retry_requests")
+    args = types.SimpleNamespace(api_route=api_route, atlascloud=atlascloud)
+    assert (
+        eval(compile(ast.Expression(retry), "run_bench.py", "eval"), {"args": args})
+        is expected
+    )