diff --git a/src/tamperbench/whitebox/evals/mt_bench/mt_bench.py b/src/tamperbench/whitebox/evals/mt_bench/mt_bench.py index e8708bb3..cf0924b5 100644 --- a/src/tamperbench/whitebox/evals/mt_bench/mt_bench.py +++ b/src/tamperbench/whitebox/evals/mt_bench/mt_bench.py @@ -5,8 +5,9 @@ from dataclasses import dataclass from functools import partial -from typing import Any +from typing import Any, cast +import pandera.polars as pa import polars as pl import torch from openai import OpenAI @@ -66,23 +67,23 @@ class MTBenchEvaluationConfig(WhiteBoxEvaluationConfig): class MTBenchInferenceSchema(InferenceSchema): """Schema for MT-Bench multi-turn inferences.""" - question_id: int - category: str - turn_1_question: str - turn_1_response: str - turn_2_question: str - turn_2_response: str + question_id: int = cast("int", pa.Field(nullable=False)) + category: str = cast("str", pa.Field(nullable=False)) + turn_1_question: str = cast("str", pa.Field(nullable=False)) + turn_1_response: str = cast("str", pa.Field(nullable=False)) + turn_2_question: str = cast("str", pa.Field(nullable=False)) + turn_2_response: str = cast("str", pa.Field(nullable=False)) class MTBenchScoreSchema(ScoreSchema): """Schema for MT-Bench scores with per-turn details.""" - question_id: int - category: str - turn_1_score: float - turn_2_score: float - judge_response_1: str - judge_response_2: str + question_id: int = cast("int", pa.Field(nullable=False)) + category: str = cast("str", pa.Field(nullable=False)) + turn_1_score: float = cast("float", pa.Field(nullable=False, coerce=True)) + turn_2_score: float = cast("float", pa.Field(nullable=False, coerce=True)) + judge_response_1: str = cast("str", pa.Field(nullable=False)) + judge_response_2: str = cast("str", pa.Field(nullable=False)) class MTBenchEvaluation(WhiteBoxEvaluation[MTBenchEvaluationConfig]): @@ -151,11 +152,11 @@ def _generate_response( max_length=2048, ) - input_ids: Tensor = encoding["input_ids"].to(model.device) - attention_mask: Tensor = encoding["attention_mask"].to(model.device) + input_ids: Tensor = cast(Tensor, encoding["input_ids"]).to(model.device) + attention_mask: Tensor = cast(Tensor, encoding["attention_mask"]).to(model.device) with torch.no_grad(): - outputs: GenerateOutput = model.generate( + outputs: GenerateOutput = model.generate( # pyright: ignore[reportCallIssue] input_ids=input_ids, attention_mask=attention_mask, max_new_tokens=max_new_tokens, @@ -261,7 +262,7 @@ def _score_row(self, row: dict[str, Any], question_map: dict[int, MTBenchQuestio return { InferenceSchema.prompt: row[InferenceSchema.prompt], InferenceSchema.response: row[InferenceSchema.response], - ScoreSchema.score: (score_1 + score_2) / 2, + ScoreSchema.score: (score_1 + score_2) / 2, # pyright: ignore[reportReturnType] "question_id": row["question_id"], "category": row["category"], "turn_1_score": score_1, @@ -305,11 +306,11 @@ def compute_results(self, scores: DataFrame[ScoreSchema]) -> DataFrame[Evaluatio scores_df = ScoreSchema.validate(scores) # Overall average score - mean_score: float = float(scores_df[ScoreSchema.score].mean()) + mean_score: float = float(scores_df[ScoreSchema.score].mean()) # pyright: ignore[reportArgumentType,reportCallIssue] # Per-turn averages - mean_turn_1: float = float(scores_df["turn_1_score"].mean()) - mean_turn_2: float = float(scores_df["turn_2_score"].mean()) + mean_turn_1: float = float(scores_df["turn_1_score"].mean()) # pyright: ignore[reportArgumentType] + mean_turn_2: float = float(scores_df["turn_2_score"].mean()) # pyright: ignore[reportArgumentType] _metrics_df: pl.DataFrame = pl.from_dict( data={ @@ -318,7 +319,7 @@ def compute_results(self, scores: DataFrame[ScoreSchema]) -> DataFrame[Evaluatio str(MetricName.MT_BENCH_TURN_1_SCORE), str(MetricName.MT_BENCH_TURN_2_SCORE), ], - EvaluationSchema.metric_value: [mean_score, mean_turn_1, mean_turn_2], + EvaluationSchema.metric_value: [mean_score, mean_turn_1, mean_turn_2], # pyright: ignore[reportArgumentType] } ) return EvaluationSchema.validate(_metrics_df)