From 725b42a76b6e132eea2602536cfb1f98617b72ea Mon Sep 17 00:00:00 2001 From: atti92 Date: Fri, 10 Apr 2026 21:16:28 +0200 Subject: [PATCH 1/2] Add scenario_docs configuration and enhance documentation checks - Introduced scenario_docs attribute in AppConfig to manage documentation requirements. - Updated get_scenario_docs function to search subdirectories for documentation files. - Implemented logging for missing documentation based on scenario_docs settings. - Added comprehensive tests for scenario documentation checks, covering various scenarios. --- .../core/execution/config_models.py | 1 + .../core/execution/dependencies.py | 17 +- tests/test_scenario_docs_check.py | 147 ++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 tests/test_scenario_docs_check.py diff --git a/src/openutm_verification/core/execution/config_models.py b/src/openutm_verification/core/execution/config_models.py index d0fd0ad2..2d4ccd1e 100644 --- a/src/openutm_verification/core/execution/config_models.py +++ b/src/openutm_verification/core/execution/config_models.py @@ -195,6 +195,7 @@ class AppConfig(StrictBaseModel): version: str = "1.0" run_id: str = "daily-conformance-check" + scenario_docs: Literal["warn", "required", "ignore"] = "warn" flight_blender: FlightBlenderConfig opensky: OpenSkyConfig air_traffic_simulator_settings: AirTrafficSimulatorSettings diff --git a/src/openutm_verification/core/execution/dependencies.py b/src/openutm_verification/core/execution/dependencies.py index a0b2a4af..46de68c7 100644 --- a/src/openutm_verification/core/execution/dependencies.py +++ b/src/openutm_verification/core/execution/dependencies.py @@ -41,7 +41,12 @@ def get_scenario_docs(scenario_id: str) -> str | None: docs_path = docs_dir / f"{scenario_id}.md" if not docs_path.exists(): - return None + # Search subdirectories for a matching file + matches = list(docs_dir.rglob(f"{scenario_id}.md")) + if len(matches) == 1: + docs_path = matches[0] + else: + return None try: return docs_path.read_text(encoding="utf-8") @@ -82,6 +87,16 @@ def scenarios() -> Iterable[str]: docs_content = get_scenario_docs(scenario_id) + if docs_content is None: + scenario_docs_mode = config.scenario_docs if hasattr(config, "scenario_docs") else "warn" + if scenario_docs_mode == "required": + logger.error(f"Scenario '{scenario_id}' has no documentation. Aborting because scenario_docs is set to 'required'.") + raise FileNotFoundError( + f"Missing documentation for scenario '{scenario_id}'. Add a '{scenario_id}.md' file under the docs/scenarios directory." + ) + elif scenario_docs_mode == "warn": + logger.warning(f"Scenario '{scenario_id}' has no documentation. Add a '{scenario_id}.md' file under the docs/scenarios directory.") + CONTEXT.set( { "scenario_id": scenario_id, diff --git a/tests/test_scenario_docs_check.py b/tests/test_scenario_docs_check.py new file mode 100644 index 00000000..b59f43d2 --- /dev/null +++ b/tests/test_scenario_docs_check.py @@ -0,0 +1,147 @@ +"""Tests for scenario documentation check (issue #101).""" + +from unittest.mock import MagicMock, patch + +import pytest + +from openutm_verification.core.execution.dependencies import get_scenario_docs, scenarios + + +class TestGetScenarioDocs: + """Tests for get_scenario_docs with recursive subdirectory search.""" + + def test_returns_content_for_existing_flat_doc(self, tmp_path): + docs_dir = tmp_path / "docs" + docs_dir.mkdir() + doc_file = docs_dir / "my_scenario.md" + doc_file.write_text("# My Scenario\nSome content.", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("my_scenario") + + assert result == "# My Scenario\nSome content." + + def test_returns_content_for_doc_in_subdirectory(self, tmp_path): + docs_dir = tmp_path / "docs" + sub_dir = docs_dir / "standard-scenarios" + sub_dir.mkdir(parents=True) + doc_file = sub_dir / "F1_happy_path.md" + doc_file.write_text("# F1 Happy Path", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("F1_happy_path") + + assert result == "# F1 Happy Path" + + def test_returns_none_when_no_docs_exist(self, tmp_path): + docs_dir = tmp_path / "docs" + docs_dir.mkdir() + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("nonexistent_scenario") + + assert result is None + + def test_returns_none_when_docs_dir_is_none(self): + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=None): + result = get_scenario_docs("any_scenario") + + assert result is None + + def test_returns_none_for_multiple_matches(self, tmp_path): + docs_dir = tmp_path / "docs" + dir1 = docs_dir / "subdir1" + dir2 = docs_dir / "subdir2" + dir1.mkdir(parents=True) + dir2.mkdir(parents=True) + (dir1 / "ambiguous.md").write_text("A", encoding="utf-8") + (dir2 / "ambiguous.md").write_text("B", encoding="utf-8") + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): + result = get_scenario_docs("ambiguous") + + assert result is None + + +class TestScenariosDocsCheck: + """Tests for documentation warnings/errors in the scenarios() generator.""" + + def _make_config(self, scenario_docs="warn", scenario_names=None): + """Create a mock config proxy with the given scenario_docs mode.""" + if scenario_names is None: + scenario_names = ["test_scenario"] + + suite_scenarios = [] + for name in scenario_names: + ss = MagicMock() + ss.name = name + suite_scenarios.append(ss) + + mock_suite = MagicMock() + mock_suite.scenarios = suite_scenarios + + suites_dict = {"default_suite": mock_suite} + + mock_config = MagicMock() + mock_config.scenario_docs = scenario_docs + mock_config.target_suites = [] + mock_config.suites = suites_dict + + return mock_config + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_warn_mode_logs_warning_for_missing_docs(self, mock_get_settings, mock_get_docs, capfd): + mock_get_settings.return_value = self._make_config(scenario_docs="warn") + mock_get_docs.return_value = None + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + mock_logger.warning.assert_any_call( + "Scenario 'test_scenario' has no documentation. Add a 'test_scenario.md' file under the docs/scenarios directory." + ) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_required_mode_raises_for_missing_docs(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="required") + mock_get_docs.return_value = None + + with patch("openutm_verification.core.execution.dependencies.logger"): + with pytest.raises(FileNotFoundError, match="Missing documentation for scenario 'test_scenario'"): + list(scenarios()) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_ignore_mode_no_warning_for_missing_docs(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="ignore") + mock_get_docs.return_value = None + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + # Should not have any warning or error about docs + for call in mock_logger.warning.call_args_list: + assert "has no documentation" not in str(call) + for call in mock_logger.error.call_args_list: + assert "has no documentation" not in str(call) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_no_warning_when_docs_exist(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="warn") + mock_get_docs.return_value = "# Some Documentation" + + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + list(scenarios()) + for call in mock_logger.warning.call_args_list: + assert "has no documentation" not in str(call) + + @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") + @patch("openutm_verification.core.execution.dependencies.get_settings") + def test_required_mode_no_error_when_docs_exist(self, mock_get_settings, mock_get_docs): + mock_get_settings.return_value = self._make_config(scenario_docs="required") + mock_get_docs.return_value = "# Documented Scenario" + + with patch("openutm_verification.core.execution.dependencies.logger"): + result = list(scenarios()) + assert result == ["test_scenario"] From 76cded6c1d6dfc365f54f2105dff51a755c5640f Mon Sep 17 00:00:00 2001 From: Attila Kobor Date: Sat, 25 Apr 2026 20:27:15 +0200 Subject: [PATCH 2/2] Address review comments: fix docs_dir guard, ambiguous matches, and dynamic path in messages --- .../core/execution/dependencies.py | 17 ++++++++++-- tests/test_scenario_docs_check.py | 26 +++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/openutm_verification/core/execution/dependencies.py b/src/openutm_verification/core/execution/dependencies.py index 46de68c7..580fdf41 100644 --- a/src/openutm_verification/core/execution/dependencies.py +++ b/src/openutm_verification/core/execution/dependencies.py @@ -39,12 +39,21 @@ def get_scenario_docs(scenario_id: str) -> str | None: if not docs_dir: return None + if not docs_dir.exists() or not docs_dir.is_dir(): + logger.warning(f"Docs directory '{docs_dir}' does not exist or is not a directory.") + return None + docs_path = docs_dir / f"{scenario_id}.md" if not docs_path.exists(): # Search subdirectories for a matching file matches = list(docs_dir.rglob(f"{scenario_id}.md")) if len(matches) == 1: docs_path = matches[0] + elif len(matches) > 1: + logger.warning( + f"Multiple documentation files found for scenario '{scenario_id}' under '{docs_dir}': {matches}. Skipping ambiguous match." + ) + return None else: return None @@ -89,13 +98,17 @@ def scenarios() -> Iterable[str]: if docs_content is None: scenario_docs_mode = config.scenario_docs if hasattr(config, "scenario_docs") else "warn" + docs_dir = get_docs_directory() if scenario_docs_mode == "required": logger.error(f"Scenario '{scenario_id}' has no documentation. Aborting because scenario_docs is set to 'required'.") raise FileNotFoundError( - f"Missing documentation for scenario '{scenario_id}'. Add a '{scenario_id}.md' file under the docs/scenarios directory." + f"Missing documentation for scenario '{scenario_id}'. " + f"Add a '{scenario_id}.md' file under '{docs_dir}' (including subdirectories)." ) elif scenario_docs_mode == "warn": - logger.warning(f"Scenario '{scenario_id}' has no documentation. Add a '{scenario_id}.md' file under the docs/scenarios directory.") + logger.warning( + f"Scenario '{scenario_id}' has no documentation. Add a '{scenario_id}.md' file under '{docs_dir}' (including subdirectories)." + ) CONTEXT.set( { diff --git a/tests/test_scenario_docs_check.py b/tests/test_scenario_docs_check.py index b59f43d2..09be4446 100644 --- a/tests/test_scenario_docs_check.py +++ b/tests/test_scenario_docs_check.py @@ -58,9 +58,23 @@ def test_returns_none_for_multiple_matches(self, tmp_path): (dir2 / "ambiguous.md").write_text("B", encoding="utf-8") with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=docs_dir): - result = get_scenario_docs("ambiguous") + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + result = get_scenario_docs("ambiguous") assert result is None + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "Multiple documentation files found" in warning_calls + + def test_returns_none_when_docs_dir_does_not_exist(self, tmp_path): + missing_dir = tmp_path / "nonexistent" + + with patch("openutm_verification.core.execution.dependencies.get_docs_directory", return_value=missing_dir): + with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: + result = get_scenario_docs("any_scenario") + + assert result is None + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "does not exist or is not a directory" in warning_calls class TestScenariosDocsCheck: @@ -89,17 +103,19 @@ def _make_config(self, scenario_docs="warn", scenario_names=None): return mock_config + @patch("openutm_verification.core.execution.dependencies.get_docs_directory") @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") @patch("openutm_verification.core.execution.dependencies.get_settings") - def test_warn_mode_logs_warning_for_missing_docs(self, mock_get_settings, mock_get_docs, capfd): + def test_warn_mode_logs_warning_for_missing_docs(self, mock_get_settings, mock_get_docs, mock_docs_dir, capfd, tmp_path): mock_get_settings.return_value = self._make_config(scenario_docs="warn") mock_get_docs.return_value = None + mock_docs_dir.return_value = tmp_path / "docs" with patch("openutm_verification.core.execution.dependencies.logger") as mock_logger: list(scenarios()) - mock_logger.warning.assert_any_call( - "Scenario 'test_scenario' has no documentation. Add a 'test_scenario.md' file under the docs/scenarios directory." - ) + warning_calls = " ".join(str(c) for c in mock_logger.warning.call_args_list) + assert "test_scenario" in warning_calls + assert "has no documentation" in warning_calls @patch("openutm_verification.core.execution.dependencies.get_scenario_docs") @patch("openutm_verification.core.execution.dependencies.get_settings")