From 59f45192e16d81c96fd952b95d6fdc2c34e7d0ff Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 30 Jul 2026 03:23:18 +0200 Subject: [PATCH 1/5] test(cicd): regression test for #228 gate metrics_summary oaeval test reports every threshold as 'metric not found' because the gate summary returned by OAEvalPlugin.run_evaluation carries no metrics_summary for the CLI threshold-evaluation path to consume. Add an end-to-end regression test (offline mock providers) that fails while the key is missing. Co-Authored-By: Kimi K3 --- tests/unit/test_cicd/test_cli_test.py | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/unit/test_cicd/test_cli_test.py b/tests/unit/test_cicd/test_cli_test.py index 0a9ea5e..739ddb8 100644 --- a/tests/unit/test_cicd/test_cli_test.py +++ b/tests/unit/test_cicd/test_cli_test.py @@ -1,5 +1,6 @@ """Unit tests for CLI test command.""" +import json import re from typer.testing import CliRunner @@ -77,3 +78,59 @@ def test_test_command_threshold_option(self): output = _strip_ansi(result.output) assert "threshold" in output.lower() assert "-t" in output + + def test_test_command_passing_threshold_is_not_reported_missing(self, tmp_path): + """Regression test for issue #228. + + A threshold that should pass must not fail with "metric not found": + the gate summary returned by ``OAEvalPlugin.run_evaluation`` must + carry ``metrics_summary`` so the CLI threshold-evaluation path can + look the metric up. Uses fully offline mock providers. + """ + dataset_path = tmp_path / "data.json" + dataset_path.write_text( + json.dumps( + [ + { + "question": "What is RAG?", + "ground_truth": "RAG combines retrieval with generation.", + "context": "RAG combines retrieval with generation.", + "ground_truth_contexts": [ + "RAG combines retrieval with generation." + ], + } + ] + ) + ) + config_path = tmp_path / "config.yaml" + config_path.write_text( + f""" +dataset: + path: {dataset_path} +llm: + provider: mock + model: mock-model +retriever: + provider: mock + settings: + collection_name: c +metrics: + retrieval: [] + generation: ["exact_match"] + performance: [] + cost: [] +report: + output: json + output_dir: {tmp_path / "reports_out"} +parallel: false +""" + ) + + result = runner.invoke( + app, + ["test", str(config_path), "-t", "exact_match:gte:0.5"], + ) + output = _strip_ansi(result.output) + + assert "not found in results" not in output + assert result.exit_code == 0, output From 0f93c0c8f85848dfffb6b70e74205a606192057f Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 30 Jul 2026 03:24:51 +0200 Subject: [PATCH 2/5] fix(cicd): populate metrics_summary in run_evaluation gate summary OAEvalPlugin.run_evaluation returned an EvaluationResult whose summary carried only gate counts, so the oaeval test threshold path read an empty metrics mapping and reported every threshold as 'metric not found' regardless of the actual scores (#228). Populate metrics_summary in the returned summary, in the same flat metric-name -> value shape the engine summary (and thus the SDK threshold path) already produces. ThresholdEvaluator stays the single consumer schema; the plugin was the incomplete producer. Co-Authored-By: Kimi K3 --- openagent_eval/cicd/plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openagent_eval/cicd/plugin.py b/openagent_eval/cicd/plugin.py index 71bb3d6..39bab6e 100644 --- a/openagent_eval/cicd/plugin.py +++ b/openagent_eval/cicd/plugin.py @@ -227,6 +227,10 @@ def run_evaluation( # Evaluate thresholds evaluator = ThresholdEvaluator(cicd_config) eval_result = evaluator.evaluate_all_gates(flat_metrics) + # Carry the metrics in the same shape the engine summary produces so + # downstream consumers (e.g. the `oaeval test` threshold path) can + # evaluate gates against them. + eval_result.summary["metrics_summary"] = flat_metrics eval_result.summary["duration_seconds"] = duration return eval_result From 838df7b39818b0e8256757d401cfe3d72ca91889 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 30 Jul 2026 03:50:27 +0200 Subject: [PATCH 3/5] test(cicd): strengthen #228 regression test with enforcement assertions Assert the passing threshold is evaluated against the real computed value (exact_match: 1.0000 gte 0.5000, gate PASS), and add a should-fail case (exact_match:gte:1.5) asserting non-zero exit and the failure reported with the real actual value. A suite that only checks the passing direction cannot notice a threshold being silently waved through. Co-Authored-By: Kimi K3 --- tests/unit/test_cicd/test_cli_test.py | 121 +++++++++++++++++--------- 1 file changed, 82 insertions(+), 39 deletions(-) diff --git a/tests/unit/test_cicd/test_cli_test.py b/tests/unit/test_cicd/test_cli_test.py index 739ddb8..35de773 100644 --- a/tests/unit/test_cicd/test_cli_test.py +++ b/tests/unit/test_cicd/test_cli_test.py @@ -16,6 +16,53 @@ def _strip_ansi(text: str) -> str: return re.sub(r"\x1b\[[0-9;]*m", "", text) +def _write_offline_config(tmp_path): + """Write a fully offline dataset + config for CLI test runs. + + The mock providers echo ``ground_truth``, so ``exact_match`` scores + exactly 1.0. Returns the config path. + """ + dataset_path = tmp_path / "data.json" + dataset_path.write_text( + json.dumps( + [ + { + "question": "What is RAG?", + "ground_truth": "RAG combines retrieval with generation.", + "context": "RAG combines retrieval with generation.", + "ground_truth_contexts": [ + "RAG combines retrieval with generation." + ], + } + ] + ) + ) + config_path = tmp_path / "config.yaml" + config_path.write_text( + f""" +dataset: + path: {dataset_path} +llm: + provider: mock + model: mock-model +retriever: + provider: mock + settings: + collection_name: c +metrics: + retrieval: [] + generation: ["exact_match"] + performance: [] + cost: [] +report: + output: json + output_dir: {tmp_path / "reports_out"} +parallel: false +""" + ) + return config_path + + class TestTestCommand: """Tests for oaeval test command.""" @@ -85,46 +132,14 @@ def test_test_command_passing_threshold_is_not_reported_missing(self, tmp_path): A threshold that should pass must not fail with "metric not found": the gate summary returned by ``OAEvalPlugin.run_evaluation`` must carry ``metrics_summary`` so the CLI threshold-evaluation path can - look the metric up. Uses fully offline mock providers. + look the metric up. Uses fully offline mock providers (the mock + echoes ``ground_truth``, so ``exact_match`` scores exactly 1.0). + + Also asserts the positive behaviour: the threshold was actually + evaluated against the real computed value, not silently waved + through. """ - dataset_path = tmp_path / "data.json" - dataset_path.write_text( - json.dumps( - [ - { - "question": "What is RAG?", - "ground_truth": "RAG combines retrieval with generation.", - "context": "RAG combines retrieval with generation.", - "ground_truth_contexts": [ - "RAG combines retrieval with generation." - ], - } - ] - ) - ) - config_path = tmp_path / "config.yaml" - config_path.write_text( - f""" -dataset: - path: {dataset_path} -llm: - provider: mock - model: mock-model -retriever: - provider: mock - settings: - collection_name: c -metrics: - retrieval: [] - generation: ["exact_match"] - performance: [] - cost: [] -report: - output: json - output_dir: {tmp_path / "reports_out"} -parallel: false -""" - ) + config_path = _write_offline_config(tmp_path) result = runner.invoke( app, @@ -133,4 +148,32 @@ def test_test_command_passing_threshold_is_not_reported_missing(self, tmp_path): output = _strip_ansi(result.output) assert "not found in results" not in output + # The threshold must be evaluated against the real value (1.0), + # and the gate must be reported as passing. + assert "exact_match: 1.0000 gte 0.5000" in output, output + assert "PASS" in output, output assert result.exit_code == 0, output + + def test_test_command_failing_threshold_is_enforced(self, tmp_path): + """Complement to the #228 regression test. + + A threshold that genuinely cannot be met (``exact_match`` scores + exactly 1.0 with the offline mock providers, so ``gte:1.5`` must + fail) must exit non-zero and report the failure with the real + actual value. This catches an implementation that auto-passes or + skips missing/unresolvable metrics: a suite that only checks the + passing direction cannot notice that enforcement stopped. + """ + config_path = _write_offline_config(tmp_path) + + result = runner.invoke( + app, + ["test", str(config_path), "-t", "exact_match:gte:1.5"], + ) + output = _strip_ansi(result.output) + + # The metric must resolve (not "not found") and the failure must + # be reported against the real computed value (1.0). + assert "not found in results" not in output + assert "exact_match: 1.0000 NOT gte 1.5000" in output, output + assert result.exit_code == 1, output From c4b73f5a4ec1bc84db2bac95c446e2564f928f1f Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 30 Jul 2026 03:53:07 +0200 Subject: [PATCH 4/5] docs(cicd): correct metrics_summary comment in run_evaluation The stored mapping is the plugin's threshold-target mapping: the engine's metric set plus the execution counts (total_items, successful_evaluations, failed_evaluations) the plugin already supports as gate targets. It is a superset of the engine summary's own metrics_summary, not the same shape. Behaviour unchanged. Co-Authored-By: Kimi K3 --- openagent_eval/cicd/plugin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openagent_eval/cicd/plugin.py b/openagent_eval/cicd/plugin.py index 39bab6e..7f14532 100644 --- a/openagent_eval/cicd/plugin.py +++ b/openagent_eval/cicd/plugin.py @@ -227,9 +227,13 @@ def run_evaluation( # Evaluate thresholds evaluator = ThresholdEvaluator(cicd_config) eval_result = evaluator.evaluate_all_gates(flat_metrics) - # Carry the metrics in the same shape the engine summary produces so - # downstream consumers (e.g. the `oaeval test` threshold path) can - # evaluate gates against them. + # Expose the plugin's threshold-target mapping so downstream + # consumers (e.g. the `oaeval test` threshold path) can evaluate + # gates against it. This is the engine's metric set plus the + # execution counts (total_items, successful_evaluations, + # failed_evaluations) that the plugin already supports as gate + # targets, so it is a superset of the engine summary's own + # `metrics_summary`, not the same set. eval_result.summary["metrics_summary"] = flat_metrics eval_result.summary["duration_seconds"] = duration From a2685071e610a0583883723c5cdeff450aded8cb Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 30 Jul 2026 04:20:03 +0200 Subject: [PATCH 5/5] fix(cicd): make run_evaluation independent of event-loop policy state asyncio.run() calls asyncio.set_event_loop(None) on completion, so after any completed asyncio.run in the process, asyncio.get_event_loop() in the main thread raises RuntimeError instead of auto-creating a loop. run_evaluation called get_event_loop() inside its broad try, so the exception was swallowed into an error summary and every gate threshold was reported as "not found in results" (issue #261). In the full-suite single-process coverage run, an earlier test completes an asyncio.run, which made the #228 regression tests fail in the Coverage CI job. Replace the get_event_loop()+is_running() probe with get_running_loop(): RuntimeError (no loop running, the normal synchronous case) runs the coroutine via asyncio.run(); a live loop keeps the existing behaviour of offloading to a thread pool with the configured timeout. Add a regression test that completes an asyncio.run() before driving 'oaeval test' and asserts the evaluation actually ran (real metric value compared, gate passes) instead of returning the error summary. Co-Authored-By: Kimi K3 --- openagent_eval/cicd/plugin.py | 23 ++++++++++++++------ tests/unit/test_cicd/test_cli_test.py | 30 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/openagent_eval/cicd/plugin.py b/openagent_eval/cicd/plugin.py index 7f14532..0da2a59 100644 --- a/openagent_eval/cicd/plugin.py +++ b/openagent_eval/cicd/plugin.py @@ -175,18 +175,29 @@ def run_evaluation( dataset_items = load_dataset_for_run(eval_config) - # Run async evaluation - loop = asyncio.get_event_loop() - if loop.is_running(): - # If we're already in an async context, create a new task + # Run async evaluation. Use get_running_loop() instead of + # get_event_loop(): after any completed asyncio.run() in the + # process, the event-loop policy holds no current loop + # (asyncio.run calls set_event_loop(None) on completion), so + # get_event_loop() raises RuntimeError before is_running() is + # ever reached — and the broad `except Exception` below then + # swallowed it into an error summary (issue #261). + # get_running_loop() only asks whether a loop is running right + # now, independent of the policy's current-loop state. + try: + asyncio.get_running_loop() + except RuntimeError: + # No loop is running: the normal synchronous case. + result = asyncio.run(engine.run(dataset_items)) + else: + # We're already inside a live async context; offload to a + # thread and run it there, preserving the timeout. import concurrent.futures with concurrent.futures.ThreadPoolExecutor() as pool: result = pool.submit( asyncio.run, engine.run(dataset_items) ).result(timeout=timeout) - else: - result = loop.run_until_complete(engine.run(dataset_items)) # Extract metrics from summary metrics = result.summary.get("metrics_summary", {}) diff --git a/tests/unit/test_cicd/test_cli_test.py b/tests/unit/test_cicd/test_cli_test.py index 35de773..fc65c2a 100644 --- a/tests/unit/test_cicd/test_cli_test.py +++ b/tests/unit/test_cicd/test_cli_test.py @@ -1,5 +1,6 @@ """Unit tests for CLI test command.""" +import asyncio import json import re @@ -177,3 +178,32 @@ def test_test_command_failing_threshold_is_enforced(self, tmp_path): assert "not found in results" not in output assert "exact_match: 1.0000 NOT gte 1.5000" in output, output assert result.exit_code == 1, output + + def test_test_command_after_completed_asyncio_run(self, tmp_path): + """Regression test for issue #261. + + ``asyncio.run()`` calls ``asyncio.set_event_loop(None)`` when it + completes, so afterwards ``asyncio.get_event_loop()`` in the main + thread raises ``RuntimeError`` instead of auto-creating a loop. + ``run_evaluation`` must not depend on that policy state: after a + completed ``asyncio.run`` in the same process, the evaluation must + still actually run and the threshold must be evaluated against the + real metrics — not swallowed into an ``error`` summary that the + CLI then reports as "not found in results". + """ + asyncio.run(asyncio.sleep(0)) + + config_path = _write_offline_config(tmp_path) + + result = runner.invoke( + app, + ["test", str(config_path), "-t", "exact_match:gte:0.5"], + ) + output = _strip_ansi(result.output) + + assert "not found in results" not in output + # The evaluation must have actually run: the threshold is + # evaluated against the real computed value (1.0) and passes. + assert "exact_match: 1.0000 gte 0.5000" in output, output + assert "PASS" in output, output + assert result.exit_code == 0, output