diff --git a/src/openutm_verification/core/execution/config_models.py b/src/openutm_verification/core/execution/config_models.py index d0fd0ad2..2d4ccd1e 100644 --- a/src/openutm_verification/core/execution/config_models.py +++ b/src/openutm_verification/core/execution/config_models.py @@ -195,6 +195,7 @@ class AppConfig(StrictBaseModel): version: str = "1.0" run_id: str = "daily-conformance-check" + scenario_docs: Literal["warn", "required", "ignore"] = "warn" flight_blender: FlightBlenderConfig opensky: OpenSkyConfig air_traffic_simulator_settings: AirTrafficSimulatorSettings diff --git a/src/openutm_verification/core/execution/dependencies.py b/src/openutm_verification/core/execution/dependencies.py index a0b2a4af..580fdf41 100644 --- a/src/openutm_verification/core/execution/dependencies.py +++ b/src/openutm_verification/core/execution/dependencies.py @@ -39,9 +39,23 @@ def get_scenario_docs(scenario_id: str) -> str | None: if not docs_dir: return None + if not docs_dir.exists() or not docs_dir.is_dir(): + logger.warning(f"Docs directory '{docs_dir}' does not exist or is not a directory.") + return None + docs_path = docs_dir / f"{scenario_id}.md" if not docs_path.exists(): - return None + # Search subdirectories for a matching file + matches = list(docs_dir.rglob(f"{scenario_id}.md")) + if len(matches) == 1: + docs_path = matches[0] + elif len(matches) > 1: + logger.warning( + f"Multiple documentation files found for scenario '{scenario_id}' under '{docs_dir}': {matches}. Skipping ambiguous match." + ) + return None + else: + return None try: return docs_path.read_text(encoding="utf-8") @@ -82,6 +96,20 @@ def scenarios() -> Iterable[str]: docs_content = get_scenario_docs(scenario_id) + if docs_content is None: + scenario_docs_mode = config.scenario_docs if hasattr(config, "scenario_docs") else "warn" + docs_dir = get_docs_directory() + if scenario_docs_mode == "required": + logger.error(f"Scenario '{scenario_id}' has no documentation. Aborting because scenario_docs is set to 'required'.") + raise FileNotFoundError( + f"Missing documentation for scenario '{scenario_id}'. " + f"Add a '{scenario_id}.md' file under '{docs_dir}' (including subdirectories)." + ) + elif scenario_docs_mode == "warn": + logger.warning( + f"Scenario '{scenario_id}' has no documentation. Add a '{scenario_id}.md' file under '{docs_dir}' (including subdirectories)." + ) + CONTEXT.set( { "scenario_id": scenario_id, diff --git a/tests/test_scenario_docs_check.py b/tests/test_scenario_docs_check.py new file mode 100644 index 00000000..09be4446 --- /dev/null +++ b/tests/test_scenario_docs_check.py @@ -0,0 +1,163 @@ +"""Tests for scenario documentation check (issue #101).""" + +from unittest.mock import MagicMock, patch + +import pytest + +from openutm_verification.core.execution.dependencies import get_scenario_docs, scenarios + + +class TestGetScenarioDocs: + """Tests for get_scenario_docs with recursive subdirectory search.""" + + def test_returns_content_for_existing_flat_doc(self, tmp_path): + docs_dir = tmp_path / "docs" + docs_dir.mkdir() + doc_file = docs_dir / "my_scenario.md" + doc_file.write_text("# My Scenario\nSome content.", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("my_scenario") + + assert result == "# My Scenario\nSome content." + + def test_returns_content_for_doc_in_subdirectory(self, tmp_path): + docs_dir = tmp_path / "docs" + sub_dir = docs_dir / "standard-scenarios" + sub_dir.mkdir(parents=True) + doc_file = sub_dir / "F1_happy_path.md" + doc_file.write_text("# F1 Happy Path", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("F1_happy_path") + + assert result == "# F1 Happy Path" + + def test_returns_none_when_no_docs_exist(self, tmp_path): + docs_dir = tmp_path / "docs" + docs_dir.mkdir() + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("nonexistent_scenario") + + assert result is None + + def test_returns_none_when_docs_dir_is_none(self): + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=None): + result = get_scenario_docs("any_scenario") + + assert result is None + + def test_returns_none_for_multiple_matches(self, tmp_path): + docs_dir = tmp_path / "docs" + dir1 = docs_dir / "subdir1" + dir2 = docs_dir / "subdir2" + dir1.mkdir(parents=True) + dir2.mkdir(parents=True) + (dir1 / "ambiguous.md").write_text("A", encoding="utf-8") + (dir2 / "ambiguous.md").write_text("B", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + result = get_scenario_docs("ambiguous") + + assert result is None + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "Multiple documentation files found" in warning_calls + + def test_returns_none_when_docs_dir_does_not_exist(self, tmp_path): + missing_dir = tmp_path / "nonexistent" + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=missing_dir): + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + result = get_scenario_docs("any_scenario") + + assert result is None + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "does not exist or is not a directory" in warning_calls + + +class TestScenariosDocsCheck: + """Tests for documentation warnings/errors in the scenarios() generator.""" + + def _make_config(self, scenario_docs="warn", scenario_names=None): + """Create a mock config proxy with the given scenario_docs mode.""" + if scenario_names is None: + scenario_names = ["test_scenario"] + + suite_scenarios = [] + for name in scenario_names: + ss = MagicMock() + ss.name = name + suite_scenarios.append(ss) + + mock_suite = MagicMock() + mock_suite.scenarios = suite_scenarios + + suites_dict = {"default_suite": mock_suite} + + mock_config = MagicMock() + mock_config.scenario_docs = scenario_docs + mock_config.target_suites = [] + mock_config.suites = suites_dict + + return mock_config + + @patch("openutm_verification.core.execution.dependencies.get_docs_directory") + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_warn_mode_logs_warning_for_missing_docs(self, mock_get_settings, mock_get_docs, mock_docs_dir, capfd, tmp_path): + mock_get_settings.return_value = self._make_config(scenario_docs="warn") + mock_get_docs.return_value = None + mock_docs_dir.return_value = tmp_path / "docs" + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "test_scenario" in warning_calls + assert "has no documentation" in warning_calls + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_required_mode_raises_for_missing_docs(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="required") + mock_get_docs.return_value = None + + with patch("openutm_verification.core.execution.dependencies.logger"): + with pytest.raises(FileNotFoundError, match="Missing documentation for scenario 'test_scenario'"): + list(scenarios()) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_ignore_mode_no_warning_for_missing_docs(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="ignore") + mock_get_docs.return_value = None + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + # Should not have any warning or error about docs + for call in mock_logger.warning.call_args_list: + assert "has no documentation" not in str(call) + for call in mock_logger.error.call_args_list: + assert "has no documentation" not in str(call) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_no_warning_when_docs_exist(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="warn") + mock_get_docs.return_value = "# Some Documentation" + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + for call in mock_logger.warning.call_args_list: + assert "has no documentation" not in str(call) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_required_mode_no_error_when_docs_exist(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="required") + mock_get_docs.return_value = "# Documented Scenario" + + with patch("openutm_verification.core.execution.dependencies.logger"): + result = list(scenarios()) + assert result == ["test_scenario"]