From efe0972c7d096ec48929ef85c12a33d0adb8b858 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Fri, 23 Jan 2026 17:37:28 -0700 Subject: [PATCH] Fix config file allowing partial overrides for judge/synthetic_user The judge: and synthetic_user: sections in mcprobe.yaml now allow partial overrides. Previously they required provider and model fields even when the user just wanted to set extra_instructions. Changes: - Add FileLLMConfigOverride model with all optional fields - Use FileLLMConfigOverride for judge/synthetic_user in FileConfig - Add _apply_file_override method for partial config application - Export FileLLMConfigOverride from config package - Update tests to use new model This allows configs like: ```yaml llm: provider: ollama model: llama3.2 judge: extra_instructions: | Custom judge instructions... ``` --- src/mcprobe/config/__init__.py | 2 ++ src/mcprobe/config/loader.py | 54 +++++++++++++++++++++++++++++--- tests/unit/test_config_loader.py | 18 ++++++++--- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/mcprobe/config/__init__.py b/src/mcprobe/config/__init__.py index c467560..b8996ab 100644 --- a/src/mcprobe/config/__init__.py +++ b/src/mcprobe/config/__init__.py @@ -5,6 +5,7 @@ CLIOverrides, ConfigLoader, FileConfig, + FileLLMConfigOverride, MCPServerConfig, ResultsConfig, load_config, @@ -15,6 +16,7 @@ "CLIOverrides", "ConfigLoader", "FileConfig", + "FileLLMConfigOverride", "MCPServerConfig", "ResultsConfig", "load_config", diff --git a/src/mcprobe/config/loader.py b/src/mcprobe/config/loader.py index 0bb7ca8..228ebfe 100644 --- a/src/mcprobe/config/loader.py +++ b/src/mcprobe/config/loader.py @@ -95,13 +95,32 @@ class CLIOverrides(BaseModel): max_tokens: int | None = None +class FileLLMConfigOverride(BaseModel): + """Partial LLM config for component-specific overrides in config file. + + All fields are optional - only set values will override the shared llm config. + This allows users to set just extra_instructions for judge/synthetic_user + while inheriting provider, model, etc. from the shared llm section. + """ + + provider: str | None = None + model: str | None = None + temperature: float | None = Field(default=None, ge=0.0, le=2.0) + max_tokens: int | None = Field(default=None, ge=1) + api_key: SecretStr | None = None + base_url: str | None = None + context_size: int | None = Field(default=None, ge=1024) + reasoning: Literal["low", "medium", "high"] | None = None + extra_instructions: str | None = None + + class FileConfig(BaseModel): """Schema for mcprobe.yaml configuration file.""" agent: AgentConfig = Field(default_factory=AgentConfig) llm: LLMConfig | None = None - judge: LLMConfig | None = None - synthetic_user: LLMConfig | None = None + judge: FileLLMConfigOverride | None = None + synthetic_user: FileLLMConfigOverride | None = None orchestrator: OrchestratorConfig = Field(default_factory=OrchestratorConfig) results: ResultsConfig = Field(default_factory=ResultsConfig) mcp_server: MCPServerConfig | None = None @@ -244,6 +263,31 @@ def _apply_llm_config(source: LLMConfig, values: _ResolvedValues) -> None: if source.extra_instructions: values.extra_instructions.append(source.extra_instructions) + @staticmethod + def _apply_file_override(source: FileLLMConfigOverride, values: _ResolvedValues) -> None: + """Apply values from a file config override (mutates values in place). + + Only non-None values are applied, allowing partial overrides. + """ + if source.provider is not None: + values.provider = source.provider + if source.model is not None: + values.model = source.model + if source.temperature is not None: + values.temperature = source.temperature + if source.max_tokens is not None: + values.max_tokens = source.max_tokens + if source.base_url is not None: + values.base_url = source.base_url + if source.api_key: + values.api_key = source.api_key.get_secret_value() + if source.context_size is not None: + values.context_size = source.context_size + if source.reasoning is not None: + values.reasoning = source.reasoning + if source.extra_instructions: + values.extra_instructions.append(source.extra_instructions) + @staticmethod def _apply_cli_overrides(cli: CLIOverrides, values: _ResolvedValues) -> None: """Apply CLI overrides to configuration values (mutates values in place).""" @@ -322,12 +366,12 @@ def resolve_llm_config( if file_config and file_config.llm: ConfigLoader._apply_llm_config(file_config.llm, values) - # Apply component-specific config - component_config: LLMConfig | None = None + # Apply component-specific config (partial overrides) + component_config: FileLLMConfigOverride | None = None if file_config: component_config = getattr(file_config, component, None) if component_config: - ConfigLoader._apply_llm_config(component_config, values) + ConfigLoader._apply_file_override(component_config, values) # Apply scenario-level overrides if scenario_override: diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py index 2c7c497..2f8a717 100644 --- a/tests/unit/test_config_loader.py +++ b/tests/unit/test_config_loader.py @@ -7,7 +7,14 @@ import pytest from pydantic import SecretStr -from mcprobe.config import AgentConfig, CLIOverrides, ConfigLoader, FileConfig, ResultsConfig +from mcprobe.config import ( + AgentConfig, + CLIOverrides, + ConfigLoader, + FileConfig, + FileLLMConfigOverride, + ResultsConfig, +) from mcprobe.exceptions import ConfigurationError from mcprobe.models.config import LLMConfig, OrchestratorConfig @@ -275,12 +282,13 @@ def test_component_overrides_shared(self) -> None: """Component-specific config overrides shared llm config.""" file_config = FileConfig( llm=LLMConfig(provider="openai", model="gpt-4"), - judge=LLMConfig(provider="openai", model="gpt-4o"), + judge=FileLLMConfigOverride(model="gpt-4o"), ) result = ConfigLoader.resolve_llm_config(file_config, "judge") assert result.model == "gpt-4o" + assert result.provider == "openai" # Inherited from shared llm def test_cli_overrides_file_config(self) -> None: """CLI arguments override file config.""" @@ -322,14 +330,14 @@ def test_different_components_can_have_different_configs(self) -> None: """Judge and synthetic_user can have different configurations.""" file_config = FileConfig( llm=LLMConfig(provider="openai", model="gpt-4"), - judge=LLMConfig(provider="openai", model="gpt-4o"), - synthetic_user=LLMConfig(provider="ollama", model="llama3.2"), + judge=FileLLMConfigOverride(model="gpt-4o"), + synthetic_user=FileLLMConfigOverride(provider="ollama", model="llama3.2"), ) judge_config = ConfigLoader.resolve_llm_config(file_config, "judge") user_config = ConfigLoader.resolve_llm_config(file_config, "synthetic_user") - assert judge_config.provider == "openai" + assert judge_config.provider == "openai" # Inherited from shared llm assert judge_config.model == "gpt-4o" assert user_config.provider == "ollama" assert user_config.model == "llama3.2"