diff --git a/tests/unit/test_cli/test_diagnose.py b/tests/unit/test_cli/test_diagnose.py new file mode 100644 index 0000000..6a0fa05 --- /dev/null +++ b/tests/unit/test_cli/test_diagnose.py @@ -0,0 +1,278 @@ +"""Tests for the oaeval diagnose command.""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from openagent_eval.cli.context import reset_context +from openagent_eval.cli.main import app +from openagent_eval.diagnosis.models import ( + BlameTarget, + DiagnosisReport, + FailureInstance, + FailureMode, +) + +if TYPE_CHECKING: + from pathlib import Path + +runner = CliRunner() + + +def strip_ansi(text: str) -> str: + """Strip ANSI escape codes from text.""" + ansi_escape = re.compile(r"\x1b\[[0-9;]*m") + return ansi_escape.sub("", text) + + +@pytest.fixture(autouse=True) +def _reset_cli_context() -> None: + """Reset the global CLI context before and after every test. + + The CLI context (quiet/json/verbose flags) lives in a module-level + global, so leaving it dirty between tests can make later tests pass + or fail depending on run order. + """ + reset_context() + yield + reset_context() + + +@pytest.fixture +def fake_analyzer(monkeypatch: pytest.MonkeyPatch) -> dict: + """Replace DiagnosisAnalyzer so no real model or analysis code is executed. + + Yields a holder dict recording every constructed analyzer instance, its + constructor kwargs, and the DiagnosisReport returned from ``analyze``. + """ + report = DiagnosisReport( + total_items=2, + blame_summary={BlameTarget.RETRIEVAL.value: 1}, + failure_summary={FailureMode.EMPTY_RETRIEVAL.value: 1}, + failures=[ + FailureInstance( + mode=FailureMode.EMPTY_RETRIEVAL, + blame=BlameTarget.RETRIEVAL, + confidence=0.85, + reason="No contexts were retrieved.", + question="What is RAG?", + evidence={"context_count": 0}, + ), + ], + chunking_issues=[], + recommendations=["Check if the retriever is using the correct embedding model."], + overall_health=0.5, + ) + holder: dict = {"instances": [], "report": report} + + def factory(**kwargs: object) -> MagicMock: + instance = MagicMock() + instance.init_kwargs = kwargs + instance.analyze = MagicMock(return_value=holder["report"]) + holder["instances"].append(instance) + return instance + + monkeypatch.setattr( + "openagent_eval.cli.commands.diagnose.DiagnosisAnalyzer", factory + ) + return holder + + +def _write_report(path: Path, payload: object) -> None: + """Write ``payload`` as JSON to ``path``.""" + path.write_text(json.dumps(payload), encoding="utf-8") + + +# --------------------------------------------------------------------------- # +# Argument parsing and help +# --------------------------------------------------------------------------- # + + +def test_diagnose_help_lists_documented_flags() -> None: + """Help output exposes the command's public arguments and options.""" + result = runner.invoke(app, ["diagnose", "--help"]) + assert result.exit_code == 0 + output = strip_ansi(result.output) + assert "report_path" in output + for flag in ("--output", "--threshold", "--max-recs", "--verbose"): + assert flag in output + + +def test_diagnose_missing_argument_exits_two() -> None: + """Invoking diagnose without a report path is a usage error (exit 2).""" + result = runner.invoke(app, ["diagnose"]) + assert result.exit_code == 2 + assert "REPORT_PATH" in result.output or "Missing" in result.output + + +# --------------------------------------------------------------------------- # +# Input validation errors (exit code 2) +# --------------------------------------------------------------------------- # + + +def test_diagnose_missing_file_exits_two(tmp_path: Path) -> None: + """A non-existent report file surfaces as exit code 2.""" + missing = tmp_path / "missing.json" + result = runner.invoke(app, ["diagnose", str(missing)]) + assert result.exit_code == 2 + assert "not found" in strip_ansi(result.output).lower() + + +def test_diagnose_non_json_file_exits_two(tmp_path: Path) -> None: + """A report file that is not JSON surfaces as exit code 2.""" + report = tmp_path / "report.txt" + report.write_text("not json", encoding="utf-8") + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 2 + assert "json" in strip_ansi(result.output).lower() + + +def test_diagnose_invalid_json_exits_two(tmp_path: Path) -> None: + """Malformed JSON in the report file surfaces as exit code 2.""" + report = tmp_path / "report.json" + report.write_text("{not valid json", encoding="utf-8") + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 2 + assert "parse" in strip_ansi(result.output).lower() + + +# --------------------------------------------------------------------------- # +# Report-loading edge cases +# --------------------------------------------------------------------------- # + + +def test_diagnose_empty_results_exits_zero(tmp_path: Path) -> None: + """A report containing an empty list of results exits cleanly with a warning.""" + report = tmp_path / "report.json" + _write_report(report, []) + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 0, result.output + assert "no evaluation results" in strip_ansi(result.output).lower() + + +def test_diagnose_unsupported_report_format_exits_zero( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """A JSON payload that is neither a list nor a {results: ...} wrapper yields no results.""" + report = tmp_path / "report.json" + _write_report(report, {"meta": "data"}) + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 0, result.output + assert "no evaluation results" in strip_ansi(result.output).lower() + assert fake_analyzer["instances"] == [] + + +def test_diagnose_pipeline_result_wrapper_is_loaded( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """A PipelineResult-style wrapper with a 'results' key is unpacked correctly.""" + report = tmp_path / "report.json" + results = [{"question": "Q1", "answer": "A1"}] + _write_report(report, {"results": results}) + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 0, result.output + assert len(fake_analyzer["instances"]) == 1 + assert fake_analyzer["instances"][0].analyze.call_args.args[0] == results + + +# --------------------------------------------------------------------------- # +# Happy path and output formats +# --------------------------------------------------------------------------- # + + +def test_diagnose_happy_path_terminal_output( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """The default terminal output renders the diagnosis report.""" + report = tmp_path / "report.json" + _write_report(report, [{"question": "Q1", "answer": "A1"}]) + result = runner.invoke(app, ["diagnose", str(report)]) + assert result.exit_code == 0, result.output + output = strip_ansi(result.output) + assert "Diagnosis Report" in output + assert "Items analyzed: 2" in output + assert "System Health" in output + assert "Retrieval" in output + assert "Check if the retriever is using the correct embedding model" in output + + +def test_diagnose_json_output_format( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """--output json emits a JSON-serialized DiagnosisReport.""" + report = tmp_path / "report.json" + _write_report(report, [{"question": "Q1", "answer": "A1"}]) + result = runner.invoke(app, ["diagnose", str(report), "--output", "json"]) + assert result.exit_code == 0, result.output + payload = json.loads(strip_ansi(result.output)) + assert payload["total_items"] == 2 + assert payload["blame_summary"] == {BlameTarget.RETRIEVAL.value: 1} + assert payload["overall_health"] == 0.5 + + +def test_diagnose_global_json_flag_triggers_json_output( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """The global --json flag forces JSON output even when --output is omitted.""" + report = tmp_path / "report.json" + _write_report(report, [{"question": "Q1", "answer": "A1"}]) + result = runner.invoke(app, ["--json", "diagnose", str(report)]) + assert result.exit_code == 0, result.output + payload = json.loads(strip_ansi(result.output)) + assert payload["total_items"] == 2 + + +# --------------------------------------------------------------------------- # +# Option wiring +# --------------------------------------------------------------------------- # + + +def test_diagnose_threshold_and_max_recs_passed_to_analyzer( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """--threshold and --max-recs reach the DiagnosisAnalyzer constructor.""" + report = tmp_path / "report.json" + _write_report(report, [{"question": "Q1", "answer": "A1"}]) + result = runner.invoke( + app, + [ + "diagnose", + str(report), + "--threshold", + "0.7", + "--max-recs", + "3", + ], + ) + assert result.exit_code == 0, result.output + kwargs = fake_analyzer["instances"][0].init_kwargs + assert kwargs["blame_threshold"] == 0.7 + assert kwargs["max_recommendations"] == 3 + + +def test_diagnose_verbose_shows_detailed_failures( + tmp_path: Path, + fake_analyzer: dict, +) -> None: + """--verbose includes detailed failure information in terminal output.""" + report = tmp_path / "report.json" + _write_report(report, [{"question": "Q1", "answer": "A1"}]) + result = runner.invoke(app, ["diagnose", str(report), "--verbose"]) + assert result.exit_code == 0, result.output + output = strip_ansi(result.output) + assert "Detailed Failures" in output + assert "Empty Retrieval" in output + assert "No contexts were retrieved" in output + assert "What is RAG?" in output diff --git a/tests/unit/test_corpus/test_auditor.py b/tests/unit/test_corpus/test_auditor.py index d9b6ca4..4a55fbd 100644 --- a/tests/unit/test_corpus/test_auditor.py +++ b/tests/unit/test_corpus/test_auditor.py @@ -110,8 +110,22 @@ def test_build_analyzers_rejects_unknown_check_names(self): @pytest.mark.asyncio async def test_audit_with_valid_checks_still_works(self, temp_corpus): """A fully valid check list still produces a normal audit report.""" + import sys + from unittest.mock import MagicMock, patch + + import numpy as np + + # Stub the embedding model so the duplicate analyzer runs its real + # similarity pipeline without downloading weights from the Hugging + # Face hub — network access is not guaranteed in CI. + mock_st = MagicMock() + mock_st.SentenceTransformer.return_value.encode.return_value = np.array( + [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]] + ) + auditor = CorpusAuditor(checks=["staleness", "duplicate"]) - report = await auditor.audit(str(temp_corpus)) + with patch.dict(sys.modules, {"sentence_transformers": mock_st}): + report = await auditor.audit(str(temp_corpus)) assert report.total_documents == 3 assert "staleness" in report.checks_performed