save

Author: Dylan Huang

.vscode/settings.json

@@ -5,6 +5,6 @@
   "python.testing.autoTestDiscoverOnSaveEnabled": true,
   "python.defaultInterpreterPath": "./.venv/bin/python",
   "python.testing.cwd": "${workspaceFolder}",
-  "editor.defaultFormatter": "ms-python.black-formatter",
-  "cursorpyright.analysis.typeCheckingMode": "recommended"
+  "python.analysis.typeCheckingMode": "strict",
+  "python.analysis.diagnosticMode": "workspace"
 }

eval_protocol/pytest/default_pydantic_ai_rollout_processor.py

@@ -2,30 +2,32 @@
 import logging
 import types
 from typing import List, Literal
+from typing import Callable, Union
 
-from openai.types.chat.chat_completion_assistant_message_param import ChatCompletionAssistantMessageParam
-
-from eval_protocol.models import EvaluationRow, Message
-from eval_protocol.pytest.rollout_processor import RolloutProcessor
-from eval_protocol.pytest.types import RolloutProcessorConfig
 from openai.types.chat import ChatCompletion, ChatCompletionMessageParam
 from openai.types.chat.chat_completion import Choice as ChatCompletionChoice
-from pydantic_ai.models.anthropic import AnthropicModel
-from pydantic_ai.models.openai import OpenAIModel
-from pydantic_ai.models.google import GoogleModel
+from openai.types.chat.chat_completion_assistant_message_param import (
+    ChatCompletionAssistantMessageParam,
+)
 from pydantic import TypeAdapter
-from pydantic_ai.messages import ModelMessage
-from pydantic_ai._utils import generate_tool_call_id
 from pydantic_ai import Agent
-from pydantic_ai.usage import UsageLimits
+from pydantic_ai._utils import generate_tool_call_id
+from pydantic_ai.messages import ModelMessage
 from pydantic_ai.messages import (
     ModelRequest,
     SystemPromptPart,
     ToolReturnPart,
     UserPromptPart,
 )
+from pydantic_ai.models.anthropic import AnthropicModel
+from pydantic_ai.models.google import GoogleModel
+from pydantic_ai.models.openai import OpenAIModel
 from pydantic_ai.providers.openai import OpenAIProvider
-from typing import Callable, Union
+from pydantic_ai.usage import UsageLimits
+
+from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
+from eval_protocol.pytest.types import RolloutProcessorConfig
 
 logger = logging.getLogger(__name__)
 
@@ -40,17 +42,7 @@ def __init__(self, setup_agent: Union[Callable[..., Agent], Agent], usage_limits
         self.setup_agent = setup_agent
         self.usage_limits = usage_limits
 
-    def _map_litellm_to_pydantic_ai(
-        self, model_name: str
-    ) -> Literal[
-        "openai",
-        "deepseek",
-        "azure",
-        "openrouter",
-        "grok",
-        "fireworks",
-        "together",
-    ]:
+    def _map_litellm_to_pydantic_ai(self, model_name: str) -> str:
         mapping = {
             "fireworks_ai": "fireworks",
             "together_ai": "together",
@@ -61,7 +53,7 @@ def _map_litellm_to_pydantic_ai(
         model_name = model_name.removeprefix(f"{provider}/")
         if provider in mapping:
             provider = mapping[provider]
-        return provider, model_name
+        return 2
 
     def _map_litellm_to_pydantic_ai_model(self, model_name: str) -> Union[OpenAIModel, GoogleModel, AnthropicModel]:
         if model_name.startswith("anthropic/"):

pyproject.toml

@@ -65,7 +65,6 @@ dev = [
     "pytest-httpserver",
     "werkzeug>=2.0.0",
     "ruff>=0.5.0",
-    "pyright>=1.1.365",
     "transformers>=4.0.0",
     "types-setuptools",
     "types-requests",
@@ -205,38 +204,9 @@ known-first-party = ["eval_protocol"]
 combine-as-imports = true
 
 [tool.pyright]
-typeCheckingMode = "basic"
+typeCheckingMode = "recommended"
 pythonVersion = "3.10"
-reportMissingImports = "none"
-reportMissingTypeStubs = "none"
-reportMissingModuleSource = "none"
 include = ["eval_protocol", "examples", "tests"]
 exclude = ["vite-app", "vendor"]
 # Ignore diagnostics for vendored generator code
 ignore = ["versioneer.py"]
-# Relax noisy diagnostics commonly triggered in tests and dynamic libs
-reportAttributeAccessIssue = "none"
-reportCallIssue = "none"
-reportUnknownMemberType = "none"
-reportUnknownVariableType = "none"
-reportPossiblyUnboundVariable = "none"
-# Additional suppressions per request
-reportOptionalMemberAccess = "none"
-reportIndexIssue = "none"
-reportReturnType = "none"
-reportOptionalCall = "none"
-reportGeneralTypeIssues = "none"
-reportOperatorIssue = "none"
-reportOptionalSubscript = "none"
-reportUnsupportedDunderAll = "none"
-reportOptionalContextManager = "none"
-reportInvalidTypeForm = "none"
-reportRedeclaration = "none"
-reportUndefinedVariable = "none"
-reportPrivateImportUsage = "none"
-reportOptionalIterable = "none"
-# Make incompatibilities and argument types warnings instead of errors for now
-# and suppress warnings output entirely
-reportIncompatibleVariableOverride = "none"
-reportArgumentType = "none"
-reportAssignmentType = "none"