diff --git a/README.md b/README.md index 69b7486..de04e21 100644 --- a/README.md +++ b/README.md @@ -157,13 +157,14 @@ pytest tests ## Tool validation setup To automate and harmonize the invocation of different tools in the validation process, tool wrapper classes can be created. -Each tool wrapper must be derived from the base class `OSCTool`: +Tool wrappers should derive from the base class `OSCTool`: This wrapper should: - Launch the tool with the given input files (and desired parameters) - Return the `ChannelSpecification` to the tool-generated OSI output - Optionally internally post-process traces (e.g., converting formats, fix tool-specific issues) +- Optionally provide a get_version() method that returns tool version information Tools already integrated: @@ -171,10 +172,58 @@ Tools already integrated: - GTGen Simulator (via gtgen_cli) - OscSimulator -To integrate a custom tool: +To integrate a custom tool without modifying this repository: - Implement a wrapper subclass of `OSCTool` -- Register it in `osc_validation/pytest_plugin.py` → `_make_tool`: +- Expose a `create_tool(toolpath)` function from a Python module or `.py` file: + + ```python + from pathlib import Path + + from osi_utilities import ChannelSpecification + from osc_validation.tools import OSCTool + + + class YourToolWrapper(OSCTool): + def __init__(self, toolpath): + super().__init__(Path(toolpath) if toolpath else None) + + def get_version(self): + return ["your-tool 1.0"] + + def run( + self, + osc_path: Path, + odr_path: Path, + osi_output_spec: ChannelSpecification, + log_path: Path = None, + rate: float = 0.05, + ) -> ChannelSpecification: + # Launch your tool, write or convert the OSI trace, and return + # the resulting channel specification. + return osi_output_spec + + + def create_tool(toolpath): + return YourToolWrapper(toolpath) + ``` +- Pass the module to `osc-validate` with `--tool-wrapper-module`: + + ```bash + osc-validate --tool YourToolName --tool-wrapper-module your_package.your_wrapper --toolpath /path/to/your/tool + ``` + + You can also pass a local `.py` file: + + ```bash + osc-validate --tool YourToolName --tool-wrapper-module /path/to/your_wrapper.py --toolpath /path/to/your/tool + ``` + +To integrate a wrapper as a built-in tool in this repository: + +- Add the wrapper implementation to `osc_validation/tools/` +- Register it in `osc_validation/pytest_plugin.py` -> `_make_tool` +- Add the tool name to the user-facing documentation and CLI tests ```python def _make_tool(config): @@ -186,17 +235,13 @@ To integrate a custom tool: elif tool_name == "GTGen": return GTGen_Simulator(toolpath) elif tool_name == "YourToolName": - return YourToolWrapperClass(toolpath) + return YourToolWrapper(toolpath) raise ValueError("Tool not found") ``` - The pytest fixture `generate_tool_trace` then yields the `run` function callable of the selected tool wrapper: - ```python - @pytest.fixture(scope="session") - def generate_tool_trace(request): - yield request.config._osc_tool.run - ``` - Using the `generate_tool_trace` fixture in a test case function enables to inject the tool execution process into test cases. - Note that the fixture `generate_tool_trace` is a callable and accepts the corresponding function parameters of the `run` function. + +The pytest fixture `generate_tool_trace` then yields the `run` function callable of the selected tool wrapper. +Using the `generate_tool_trace` fixture in a test case function enables injecting the tool execution process into test cases. +The fixture is callable and accepts the corresponding function parameters of the wrapper's `run` function. ## Test case design diff --git a/osc_validation/cli.py b/osc_validation/cli.py index 1c6fc0c..d0346be 100644 --- a/osc_validation/cli.py +++ b/osc_validation/cli.py @@ -14,6 +14,13 @@ def _resolve_from_cwd(path: str) -> str: return str(Path(path).resolve()) +def _resolve_wrapper_module(value: str) -> str: + wrapper_path = Path(value) + if wrapper_path.suffix == ".py" or wrapper_path.exists(): + return _resolve_from_cwd(value) + return value + + def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser = argparse.ArgumentParser( prog="osc-validate", @@ -22,8 +29,20 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument( "--tool", required=True, - choices=["ESMini", "GTGen", "OscSimulator"], - help="Tool to validate.", + metavar="TOOL", + help=( + "Tool to validate. Built-in tools: ESMini, GTGen, OscSimulator. " + "Other tools require --tool-wrapper-module." + ), + ) + parser.add_argument( + "--tool-wrapper-module", + default=None, + metavar="MODULE_OR_PATH", + help=( + "Python module name or .py file path providing " + "create_tool(toolpath) for a custom tool wrapper." + ), ) parser.add_argument( "--toolpath", @@ -87,6 +106,10 @@ def _pytest_args(args: argparse.Namespace, validation_dir: Path) -> list[str]: if args.toolpath is not None: pytest_args.extend(["--toolpath", _resolve_from_cwd(args.toolpath)]) + if args.tool_wrapper_module is not None: + pytest_args.extend( + ["--tool-wrapper-module", _resolve_wrapper_module(args.tool_wrapper_module)] + ) if args.test_profile is not None: pytest_args.extend(["--test-profile", _resolve_from_cwd(args.test_profile)]) if args.html is not None: diff --git a/osc_validation/pytest_plugin.py b/osc_validation/pytest_plugin.py index 287bb4e..38f2181 100644 --- a/osc_validation/pytest_plugin.py +++ b/osc_validation/pytest_plugin.py @@ -7,7 +7,8 @@ Responsibilities ---------------- -* Register CLI options (``--tool``, ``--toolpath``, ``--test-profile``). +* Register CLI options (``--tool``, ``--toolpath``, + ``--tool-wrapper-module``, ``--test-profile``). * Initialise the tool under test and expose it via the ``generate_tool_trace`` session fixture. * Expose optional QC OSI trace checking via the @@ -22,8 +23,12 @@ """ import datetime +import importlib +import importlib.util import pathlib from urllib.parse import urlparse +import sys +import uuid import pytest @@ -44,9 +49,17 @@ class UnknownToolError(ValueError): pass +class ToolWrapperError(ValueError): + pass + + def _make_tool(config): tool_name = config.getoption("--tool") toolpath = config.getoption("--toolpath") + wrapper_module = config.getoption("--tool-wrapper-module") + + if wrapper_module is not None: + return _make_custom_tool(wrapper_module, toolpath) if tool_name == "ESMini": return ESMini(toolpath) @@ -57,12 +70,85 @@ def _make_tool(config): raise UnknownToolError(f"Unknown tool: {tool_name}") +def _make_custom_tool(wrapper_module: str, toolpath: str | None): + module = _load_wrapper_module(wrapper_module) + create_tool = getattr(module, "create_tool", None) + if not callable(create_tool): + raise ToolWrapperError( + f"Custom tool wrapper module '{wrapper_module}' must define " + "a callable create_tool(toolpath)." + ) + + try: + tool = create_tool(toolpath) + except Exception as exc: + raise ToolWrapperError( + f"Custom tool wrapper module '{wrapper_module}' failed while " + f"creating the tool: {exc}" + ) from exc + + if not callable(getattr(tool, "run", None)): + raise ToolWrapperError( + f"Custom tool wrapper module '{wrapper_module}' returned an object " + "without a callable run method." + ) + return tool + + +def _load_wrapper_module(wrapper_module: str): + module_path = pathlib.Path(wrapper_module) + if module_path.suffix == ".py" or module_path.exists(): + return _load_wrapper_module_from_path(module_path, wrapper_module) + + try: + return importlib.import_module(wrapper_module) + except Exception as exc: + raise ToolWrapperError( + f"Could not import custom tool wrapper module '{wrapper_module}': {exc}" + ) from exc + + +def _load_wrapper_module_from_path(module_path: pathlib.Path, wrapper_module: str): + if not module_path.exists(): + raise ToolWrapperError( + f"Custom tool wrapper module file does not exist: {wrapper_module}" + ) + module_name = f"_osc_validation_custom_tool_{uuid.uuid4().hex}" + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise ToolWrapperError( + f"Could not load custom tool wrapper module from file: {wrapper_module}" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception as exc: + sys.modules.pop(module_name, None) + raise ToolWrapperError( + f"Could not load custom tool wrapper module from file " + f"'{wrapper_module}': {exc}" + ) from exc + return module + + +def _get_tool_version(tool): + get_version = getattr(tool, "get_version", None) + if not callable(get_version): + return ["unknown version"] + return get_version() + + def pytest_addoption(parser): group = parser.getgroup("OSC Validation Suite") group.addoption( "--tool", action="store", - help="Tool to Validate: ESMini, GTGen, OscSimulator", + metavar="TOOL", + help=( + "Tool to validate. Built-in tools: ESMini, GTGen, OscSimulator. " + "Other tools require --tool-wrapper-module" + ), ) group.addoption( "--toolpath", @@ -71,6 +157,16 @@ def pytest_addoption(parser): metavar="PATH", help="Path to the tool to validate", ) + group.addoption( + "--tool-wrapper-module", + action="store", + default=None, + metavar="MODULE_OR_PATH", + help=( + "Python module name or .py file path providing " + "create_tool(toolpath) for a custom tool wrapper" + ), + ) group.addoption( "--test-profile", action="store", @@ -119,10 +215,10 @@ def pytest_configure(config): try: tool = _make_tool(config) - except (UnknownToolError, FileNotFoundError) as exc: + except (UnknownToolError, ToolWrapperError, FileNotFoundError) as exc: raise pytest.UsageError(str(exc)) from exc config._osc_tool = tool - config._osc_tool_version = tool.get_version() + config._osc_tool_version = _get_tool_version(tool) profile_path = config.getoption("--test-profile") if profile_path is not None: diff --git a/tests/test_cli_entrypoints.py b/tests/test_cli_entrypoints.py index d558df9..af259f8 100644 --- a/tests/test_cli_entrypoints.py +++ b/tests/test_cli_entrypoints.py @@ -179,6 +179,77 @@ def fake_subprocess_run(command, check, cwd): assert f"--junitxml={Path('validation-results.xml').resolve()}" in captured_command +def test_osc_validate_main_forwards_tool_wrapper_module(monkeypatch): + captured_command = None + validation_dir = Path("installed-validation") + + def fake_subprocess_run(command, check, cwd): + nonlocal captured_command + captured_command = command + assert check is False + assert cwd == validation_dir + return type("CompletedProcess", (), {"returncode": 0})() + + monkeypatch.setattr(cli_module.subprocess, "run", fake_subprocess_run) + monkeypatch.setattr(cli_module, "_validation_dir", lambda: validation_dir) + monkeypatch.setattr( + cli_module.Path, + "exists", + lambda self: self.name in {"pytest.ini", "scenario"}, + ) + + assert ( + cli_module.main( + [ + "--tool", + "CustomTool", + "--tool-wrapper-module", + "custom_package.wrapper", + ] + ) + == 0 + ) + + assert "--tool" in captured_command + assert "CustomTool" in captured_command + assert "--tool-wrapper-module" in captured_command + assert "custom_package.wrapper" in captured_command + + +def test_osc_validate_main_resolves_tool_wrapper_file(monkeypatch): + captured_command = None + validation_dir = Path("installed-validation") + + def fake_subprocess_run(command, check, cwd): + nonlocal captured_command + captured_command = command + assert check is False + assert cwd == validation_dir + return type("CompletedProcess", (), {"returncode": 0})() + + monkeypatch.setattr(cli_module.subprocess, "run", fake_subprocess_run) + monkeypatch.setattr(cli_module, "_validation_dir", lambda: validation_dir) + monkeypatch.setattr( + cli_module.Path, + "exists", + lambda self: self.name in {"pytest.ini", "scenario"}, + ) + + assert ( + cli_module.main( + [ + "--tool", + "CustomTool", + "--tool-wrapper-module", + "custom_wrapper.py", + ] + ) + == 0 + ) + + assert str(Path("custom_wrapper.py").resolve()) in captured_command + + def test_osc_validate_main_rejects_positional_test_paths(monkeypatch): monkeypatch.setattr( cli_module.subprocess, diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py new file mode 100644 index 0000000..e2bfd8d --- /dev/null +++ b/tests/test_pytest_plugin.py @@ -0,0 +1,162 @@ +from pathlib import Path +import sys +import tempfile +import types + +import pytest + +import osc_validation.pytest_plugin as plugin + + +class FakeConfig: + def __init__(self, **options): + self.options = options + + def getoption(self, name): + return self.options.get(name) + + +class FakeBuiltInTool: + def __init__(self, toolpath): + self.tool_path = Path(toolpath) + + def run(self, osc_path, odr_path, osi_output_spec): + return osi_output_spec + + def get_version(self): + return ["fake built-in"] + + +def _write_temp_module(source: str) -> Path: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + suffix=".py", + prefix="osc_validation_custom_tool_", + dir=Path.cwd(), + delete=False, + ) as temp_file: + temp_file.write(source) + return Path(temp_file.name) + + +def test_make_tool_uses_builtin_dispatch(monkeypatch): + monkeypatch.setattr(plugin, "ESMini", FakeBuiltInTool) + + tool = plugin._make_tool( + FakeConfig( + **{ + "--tool": "ESMini", + "--toolpath": "C:/tools/esmini.exe", + "--tool-wrapper-module": None, + } + ) + ) + + assert isinstance(tool, FakeBuiltInTool) + assert tool.tool_path == Path("C:/tools/esmini.exe") + + +def test_make_tool_loads_custom_wrapper_by_import_name(monkeypatch): + class CustomTool: + def __init__(self, toolpath): + self.tool_path = toolpath + + def run(self, osc_path, odr_path, osi_output_spec): + return osi_output_spec + + def get_version(self): + return ["custom 1.0"] + + module = types.ModuleType("custom_wrapper") + module.create_tool = CustomTool + monkeypatch.setitem(sys.modules, "custom_wrapper", module) + + tool = plugin._make_tool( + FakeConfig( + **{ + "--tool": "CustomTool", + "--toolpath": "C:/tools/custom.exe", + "--tool-wrapper-module": "custom_wrapper", + } + ) + ) + + assert tool.tool_path == "C:/tools/custom.exe" + assert tool.get_version() == ["custom 1.0"] + + +def test_make_tool_loads_custom_wrapper_by_file_path(): + wrapper = _write_temp_module( + """ +class CustomTool: + def __init__(self, toolpath): + self.tool_path = toolpath + + def run(self, osc_path, odr_path, osi_output_spec): + return osi_output_spec + +def create_tool(toolpath): + return CustomTool(toolpath) +""" + ) + + try: + tool = plugin._make_tool( + FakeConfig( + **{ + "--tool": "CustomTool", + "--toolpath": "C:/tools/custom.exe", + "--tool-wrapper-module": str(wrapper), + } + ) + ) + + assert tool.tool_path == "C:/tools/custom.exe" + assert plugin._get_tool_version(tool) == ["unknown version"] + finally: + wrapper.unlink(missing_ok=True) + + +def test_custom_wrapper_requires_create_tool(): + wrapper = _write_temp_module("VALUE = 1\n") + + try: + with pytest.raises(plugin.ToolWrapperError, match="create_tool"): + plugin._make_tool( + FakeConfig( + **{ + "--tool": "CustomTool", + "--toolpath": None, + "--tool-wrapper-module": str(wrapper), + } + ) + ) + finally: + wrapper.unlink(missing_ok=True) + + +def test_custom_wrapper_requires_callable_run(): + wrapper = _write_temp_module( + """ +class CustomTool: + pass + +def create_tool(toolpath): + return CustomTool() +""" + ) + + try: + with pytest.raises(plugin.ToolWrapperError, match="callable run"): + plugin._make_tool( + FakeConfig( + **{ + "--tool": "CustomTool", + "--toolpath": None, + "--tool-wrapper-module": str(wrapper), + } + ) + ) + finally: + wrapper.unlink(missing_ok=True)