diff --git a/README.md b/README.md index 168d00c..e89b84d 100755 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ MCP-Bench is a comprehensive evaluation framework designed to assess Large Langu *Overall Score represents the average performance across all evaluation dimensions including rule-based schema understanding, LLM-judged (o4-mini as judge model) task completion, tool usage, and planning effectiveness. Scores are averaged across single-server and multi-server settings.* +LLM judge prompts use a 0-10 raw scoring rubric. MCP-Bench normalizes LLM judge subdimension and aggregate scores to the 0-1 range in result JSON, while rule-based rate metrics such as schema compliance are already reported on the 0-1 scale. + ## Quick Start ### Installation diff --git a/benchmark/evaluator.py b/benchmark/evaluator.py index c670e3e..a5729d7 100644 --- a/benchmark/evaluator.py +++ b/benchmark/evaluator.py @@ -15,6 +15,10 @@ import jsonschema from jsonschema import ValidationError import config.config_loader as config_loader +from benchmark.score_normalization import ( + LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS, + normalize_raw_llm_judge_scores, +) logger = logging.getLogger(__name__) @@ -384,7 +388,12 @@ def _calculate_average_scores(self, all_scores: List[Dict[str, Any]]) -> Dict[st logger.warning(f"No valid scores found for {field}, using 0") averaged_result[field] = 0 - # Recalculate aggregate scores from averaged subdimension scores + averaged_result = normalize_raw_llm_judge_scores( + averaged_result, + LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS, + ) + + # Recalculate aggregate scores from normalized subdimension scores task_completion_scores = [ averaged_result['task_fulfillment'], averaged_result['grounding'] @@ -880,15 +889,19 @@ async def _perform_evaluation(self, task: str, final_solution: str, logger.info(f" Parsing time: {parse_time:.3f}s") logger.debug(f"Parsed result: {result}") - # Extract 6 subdimension scores - task_fulfillment = result.get('task_fulfillment') - grounding = result.get('grounding') + # Extract and normalize 6 raw subdimension scores from 0-10 to 0-1. + normalized_result = normalize_raw_llm_judge_scores( + result, + LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS, + ) + task_fulfillment = normalized_result.get('task_fulfillment') + grounding = normalized_result.get('grounding') - tool_appropriateness = result.get('tool_appropriateness') - parameter_accuracy = result.get('parameter_accuracy') + tool_appropriateness = normalized_result.get('tool_appropriateness') + parameter_accuracy = normalized_result.get('parameter_accuracy') - dependency_awareness = result.get('dependency_awareness') - parallelism_and_efficiency = result.get('parallelism_and_efficiency') + dependency_awareness = normalized_result.get('dependency_awareness') + parallelism_and_efficiency = normalized_result.get('parallelism_and_efficiency') # Calculate aggregate scores (2 scores per category) task_completion_scores = [task_fulfillment, grounding] diff --git a/benchmark/score_normalization.py b/benchmark/score_normalization.py new file mode 100644 index 0000000..73b10f7 --- /dev/null +++ b/benchmark/score_normalization.py @@ -0,0 +1,38 @@ +""" +Utilities for normalizing MCP-Bench LLM judge scores. + +The judge prompt asks for 0-10 scores. Benchmark result JSON stores these +scores on a 0-1 scale so they are comparable with the other rate metrics. +""" + +from numbers import Real +from typing import Any, Dict, Iterable + + +LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS = ( + "task_fulfillment", + "grounding", + "tool_appropriateness", + "parameter_accuracy", + "dependency_awareness", + "parallelism_and_efficiency", +) + + +def normalize_raw_llm_judge_score(value: Any) -> Any: + """Convert a raw 0-10 LLM judge score to the benchmark's 0-1 scale.""" + if isinstance(value, bool) or not isinstance(value, Real): + return value + return value / 10 + + +def normalize_raw_llm_judge_scores( + scores: Dict[str, Any], + fields: Iterable[str] = LLM_JUDGE_SUBDIMENSION_SCORE_FIELDS, +) -> Dict[str, Any]: + """Return a copy with raw LLM judge score fields normalized to 0-1.""" + normalized = dict(scores) + for field in fields: + if field in normalized: + normalized[field] = normalize_raw_llm_judge_score(normalized[field]) + return normalized diff --git a/tests/test_score_normalization.py b/tests/test_score_normalization.py new file mode 100644 index 0000000..73b5464 --- /dev/null +++ b/tests/test_score_normalization.py @@ -0,0 +1,115 @@ +import json +import unittest + +from benchmark.evaluator import LLMJudge, TaskEvaluator +from benchmark.score_normalization import normalize_raw_llm_judge_scores + + +class StubLLM: + async def get_completion(self, system_prompt, user_prompt, max_tokens): + return json.dumps({ + "task_fulfillment_reasoning": "ok", + "grounding_reasoning": "ok", + "tool_appropriateness_reasoning": "ok", + "parameter_accuracy_reasoning": "ok", + "dependency_awareness_reasoning": "ok", + "parallelism_efficiency_reasoning": "ok", + "task_fulfillment": 8, + "grounding": 6, + "tool_appropriateness": 10, + "parameter_accuracy": 4, + "dependency_awareness": 7, + "parallelism_and_efficiency": 3, + }) + + def clean_and_parse_json(self, raw_json): + return json.loads(raw_json) + + +class ScoreNormalizationTest(unittest.TestCase): + def test_normalizes_raw_llm_judge_scores_to_zero_one_scale(self): + normalized = normalize_raw_llm_judge_scores({ + "task_fulfillment": 1, + "grounding": 8.5, + "tool_appropriateness": 10, + "parameter_accuracy": 0, + "input_schema_compliance": 0.95, + "task_fulfillment_reasoning": "raw judge score", + }) + + self.assertEqual(normalized["task_fulfillment"], 0.1) + self.assertEqual(normalized["grounding"], 0.85) + self.assertEqual(normalized["tool_appropriateness"], 1.0) + self.assertEqual(normalized["parameter_accuracy"], 0) + self.assertEqual(normalized["input_schema_compliance"], 0.95) + self.assertEqual(normalized["task_fulfillment_reasoning"], "raw judge score") + + def test_stability_average_scores_are_normalized_before_aggregation(self): + judge = LLMJudge(llm_provider=None) + + averaged = judge._calculate_average_scores([ + { + "task_fulfillment": 8, + "grounding": 6, + "tool_appropriateness": 10, + "parameter_accuracy": 4, + "dependency_awareness": 7, + "parallelism_and_efficiency": 3, + "task_completion_analysis": "first", + }, + { + "task_fulfillment": 6, + "grounding": 4, + "tool_appropriateness": 8, + "parameter_accuracy": 6, + "dependency_awareness": 5, + "parallelism_and_efficiency": 5, + "task_completion_analysis": "second", + }, + ]) + + self.assertAlmostEqual(averaged["task_fulfillment"], 0.7) + self.assertAlmostEqual(averaged["grounding"], 0.5) + self.assertAlmostEqual(averaged["tool_appropriateness"], 0.9) + self.assertAlmostEqual(averaged["parameter_accuracy"], 0.5) + self.assertAlmostEqual(averaged["dependency_awareness"], 0.6) + self.assertAlmostEqual(averaged["parallelism_and_efficiency"], 0.4) + self.assertAlmostEqual(averaged["task_completion_score"], 0.6) + self.assertAlmostEqual(averaged["tool_selection_score"], 0.7) + self.assertAlmostEqual( + averaged["planning_effectiveness_and_efficiency_score"], + 0.5, + ) + self.assertEqual(averaged["task_completion_analysis"], "first") + + +class TaskEvaluatorScoreNormalizationTest(unittest.IsolatedAsyncioTestCase): + async def test_task_evaluator_returns_normalized_llm_judge_scores(self): + evaluator = TaskEvaluator(StubLLM()) + + evaluation = await evaluator.evaluate( + task="test task", + execution_results=[], + final_solution="done", + total_rounds=1, + available_tools={}, + planning_json_compliance=1.0, + ) + + self.assertAlmostEqual(evaluation["task_fulfillment"], 0.8) + self.assertAlmostEqual(evaluation["grounding"], 0.6) + self.assertAlmostEqual(evaluation["tool_appropriateness"], 1.0) + self.assertAlmostEqual(evaluation["parameter_accuracy"], 0.4) + self.assertAlmostEqual(evaluation["dependency_awareness"], 0.7) + self.assertAlmostEqual(evaluation["parallelism_and_efficiency"], 0.3) + self.assertAlmostEqual(evaluation["task_completion_score"], 0.7) + self.assertAlmostEqual(evaluation["tool_selection_score"], 0.7) + self.assertAlmostEqual( + evaluation["planning_effectiveness_and_efficiency_score"], + 0.5, + ) + self.assertIsNone(evaluation["input_schema_compliance"]) + + +if __name__ == "__main__": + unittest.main()