From fa4409f2e814a891ceb23056ce037cd97d0f7ca9 Mon Sep 17 00:00:00 2001 From: Sri Roopa Ramesh Babu Date: Fri, 15 May 2026 14:05:28 -0400 Subject: [PATCH] removing temperature param and pass threshold. --- eval/system_azure_openai_lseval.yaml | 1 - eval/system_openai_lseval.yaml | 1 - eval/system_rhelai_vllm_lseval.yaml | 1 - eval/system_rhoai_vllm_lseval.yaml | 1 - eval/system_watsonx_lseval.yaml | 1 - eval/troubleshooting/system.yaml | 1 - tests/e2e/evaluation/test_lseval_periodic.py | 37 +++++++++++--------- 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/eval/system_azure_openai_lseval.yaml b/eval/system_azure_openai_lseval.yaml index 79fd1df91..0a7b0a60a 100644 --- a/eval/system_azure_openai_lseval.yaml +++ b/eval/system_azure_openai_lseval.yaml @@ -15,7 +15,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 0.0 max_tokens: 512 timeout: 300 num_retries: 3 diff --git a/eval/system_openai_lseval.yaml b/eval/system_openai_lseval.yaml index a977ed66b..a987b66d1 100644 --- a/eval/system_openai_lseval.yaml +++ b/eval/system_openai_lseval.yaml @@ -16,7 +16,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 1.0 max_tokens: 512 timeout: 300 num_retries: 3 diff --git a/eval/system_rhelai_vllm_lseval.yaml b/eval/system_rhelai_vllm_lseval.yaml index 43b89a36a..fb6bcdec7 100644 --- a/eval/system_rhelai_vllm_lseval.yaml +++ b/eval/system_rhelai_vllm_lseval.yaml @@ -18,7 +18,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 0.0 max_tokens: 512 timeout: 300 num_retries: 3 diff --git a/eval/system_rhoai_vllm_lseval.yaml b/eval/system_rhoai_vllm_lseval.yaml index f71688e21..746a3924c 100644 --- a/eval/system_rhoai_vllm_lseval.yaml +++ b/eval/system_rhoai_vllm_lseval.yaml @@ -18,7 +18,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 0.0 max_tokens: 512 timeout: 300 num_retries: 3 diff --git a/eval/system_watsonx_lseval.yaml b/eval/system_watsonx_lseval.yaml index 2abcfdc10..0c5b888a1 100644 --- a/eval/system_watsonx_lseval.yaml +++ b/eval/system_watsonx_lseval.yaml @@ -15,7 +15,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 0.0 max_tokens: 512 timeout: 300 num_retries: 3 diff --git a/eval/troubleshooting/system.yaml b/eval/troubleshooting/system.yaml index b869c2a77..432b016f0 100644 --- a/eval/troubleshooting/system.yaml +++ b/eval/troubleshooting/system.yaml @@ -14,7 +14,6 @@ core: llm: provider: "openai" model: "gpt-5-mini" - temperature: 0.0 max_tokens: 5000 timeout: 300 num_retries: 3 diff --git a/tests/e2e/evaluation/test_lseval_periodic.py b/tests/e2e/evaluation/test_lseval_periodic.py index 9c1c25b51..ea7d8d444 100644 --- a/tests/e2e/evaluation/test_lseval_periodic.py +++ b/tests/e2e/evaluation/test_lseval_periodic.py @@ -41,8 +41,6 @@ import pytest import yaml -MAX_EVAL_ERROR_RATE_PCT = 10.0 - # LSEval periodic provider matrix (operator-backed backends under test) _LSEVAL_PERIODIC_PROVIDERS = ("openai", "watsonx", "azure_openai") @@ -189,17 +187,25 @@ def _run_lseval(eval_data: Path, out_dir: Path, system_config: Path) -> None: summary_json = json.load(fh) overall = summary_json["summary_stats"]["overall"] - if overall["error_rate"] > MAX_EVAL_ERROR_RATE_PCT: - judge_tokens = overall.get("total_judge_llm_tokens", -1) - judge_detail = ( - "0 → OLS calls failed before judge was reached" - if judge_tokens == 0 - else "judge was called" - ) - print( - f"\n--- ERROR DIAGNOSTICS ---\n" - f"Judge LLM tokens used: {judge_tokens} ({judge_detail})\n" - ) + error_rate = overall["error_rate"] + total = overall["TOTAL"] + errors = overall["ERROR"] + passed = total - errors + + judge_tokens = overall.get("total_judge_llm_tokens", -1) + judge_detail = ( + "0 → OLS calls failed before judge was reached" + if judge_tokens == 0 + else "judge was called" + ) + print( + f"\n--- EVAL SUMMARY ---\n" + f"Total={total} Passed={passed} Errors={errors} " + f"error_rate={error_rate:.1f}%\n" + f"Judge LLM tokens used: {judge_tokens} ({judge_detail})\n" + ) + + if errors: with open(csv_files[0], encoding="utf-8") as fh: reader = csv.DictReader(fh) error_rows = [r for r in reader if r.get("result") == "ERROR"] @@ -210,10 +216,7 @@ def _run_lseval(eval_data: Path, out_dir: Path, system_config: Path) -> None: f" turn={row.get('turn_id','?')} reason={row.get('reason','?')[:200]}" ) - assert overall["error_rate"] <= MAX_EVAL_ERROR_RATE_PCT, ( - f"{overall['ERROR']}/{overall['TOTAL']} evaluations errored " - f"(error_rate={overall['error_rate']:.1f}% > threshold {MAX_EVAL_ERROR_RATE_PCT}%)." - ) + assert passed > 0, f"All {total} evaluations errored — zero successful results." @pytest.mark.lseval