Allow external tool wrapper loading

README.md

@@ -157,24 +157,73 @@ pytest tests
 ## Tool validation setup
 
 To automate and harmonize the invocation of different tools in the validation process, tool wrapper classes can be created.
-Each tool wrapper must be derived from the base class `OSCTool`:
+Tool wrappers should derive from the base class `OSCTool`:
 
 This wrapper should:
 
 - Launch the tool with the given input files (and desired parameters)
 - Return the `ChannelSpecification` to the tool-generated OSI output
 - Optionally internally post-process traces (e.g., converting formats, fix tool-specific issues)
+- Optionally provide a get_version() method that returns tool version information
 
 Tools already integrated:
 
 - ESMini
 - GTGen Simulator (via gtgen_cli)
 - OscSimulator
 
-To integrate a custom tool:
+To integrate a custom tool without modifying this repository:
 
 - Implement a wrapper subclass of `OSCTool`
-- Register it in `osc_validation/pytest_plugin.py` → `_make_tool`:
+- Expose a `create_tool(toolpath)` function from a Python module or `.py` file:
+
+    ```python
+    from pathlib import Path
+
+    from osi_utilities import ChannelSpecification
+    from osc_validation.tools import OSCTool
+
+
+    class YourToolWrapper(OSCTool):
+        def __init__(self, toolpath):
+            super().__init__(Path(toolpath) if toolpath else None)
+
+        def get_version(self):
+            return ["your-tool 1.0"]
+
+        def run(
+            self,
+            osc_path: Path,
+            odr_path: Path,
+            osi_output_spec: ChannelSpecification,
+            log_path: Path = None,
+            rate: float = 0.05,
+        ) -> ChannelSpecification:
+            # Launch your tool, write or convert the OSI trace, and return
+            # the resulting channel specification.
+            return osi_output_spec
+
+
+    def create_tool(toolpath):
+        return YourToolWrapper(toolpath)
+    ```
+- Pass the module to `osc-validate` with `--tool-wrapper-module`:
+
+    ```bash
+    osc-validate --tool YourToolName --tool-wrapper-module your_package.your_wrapper --toolpath /path/to/your/tool
+    ```
+
+    You can also pass a local `.py` file:
+
+    ```bash
+    osc-validate --tool YourToolName --tool-wrapper-module /path/to/your_wrapper.py --toolpath /path/to/your/tool
+    ```
+
+To integrate a wrapper as a built-in tool in this repository:
+
+- Add the wrapper implementation to `osc_validation/tools/`
+- Register it in `osc_validation/pytest_plugin.py` -> `_make_tool`
+- Add the tool name to the user-facing documentation and CLI tests
 
     ```python
     def _make_tool(config):
@@ -186,17 +235,13 @@ To integrate a custom tool:
         elif tool_name == "GTGen":
             return GTGen_Simulator(toolpath)
         elif tool_name == "YourToolName":
-            return YourToolWrapperClass(toolpath)
+            return YourToolWrapper(toolpath)
         raise ValueError("Tool not found")
     ```
-    The pytest fixture `generate_tool_trace` then yields the `run` function callable of the selected tool wrapper:
-    ```python
-    @pytest.fixture(scope="session")
-    def generate_tool_trace(request):
-        yield request.config._osc_tool.run
-    ```
-    Using the `generate_tool_trace` fixture in a test case function enables to inject the tool execution process into test cases.
-    Note that the fixture `generate_tool_trace` is a callable and accepts the corresponding function parameters of the `run` function.
+
+The pytest fixture `generate_tool_trace` then yields the `run` function callable of the selected tool wrapper.
+Using the `generate_tool_trace` fixture in a test case function enables injecting the tool execution process into test cases.
+The fixture is callable and accepts the corresponding function parameters of the wrapper's `run` function.
 
 ## Test case design
 

osc_validation/cli.py

@@ -14,6 +14,13 @@ def _resolve_from_cwd(path: str) -> str:
     return str(Path(path).resolve())
 
 
+def _resolve_wrapper_module(value: str) -> str:
+    wrapper_path = Path(value)
+    if wrapper_path.suffix == ".py" or wrapper_path.exists():
+        return _resolve_from_cwd(value)
+    return value
+
+
 def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         prog="osc-validate",
@@ -22,8 +29,20 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     parser.add_argument(
         "--tool",
         required=True,
-        choices=["ESMini", "GTGen", "OscSimulator"],
-        help="Tool to validate.",
+        metavar="TOOL",
+        help=(
+            "Tool to validate. Built-in tools: ESMini, GTGen, OscSimulator. "
+            "Other tools require --tool-wrapper-module."
+        ),
+    )
+    parser.add_argument(
+        "--tool-wrapper-module",
+        default=None,
+        metavar="MODULE_OR_PATH",
+        help=(
+            "Python module name or .py file path providing "
+            "create_tool(toolpath) for a custom tool wrapper."
+        ),
     )
     parser.add_argument(
         "--toolpath",
@@ -87,6 +106,10 @@ def _pytest_args(args: argparse.Namespace, validation_dir: Path) -> list[str]:
 
     if args.toolpath is not None:
         pytest_args.extend(["--toolpath", _resolve_from_cwd(args.toolpath)])
+    if args.tool_wrapper_module is not None:
+        pytest_args.extend(
+            ["--tool-wrapper-module", _resolve_wrapper_module(args.tool_wrapper_module)]
+        )
     if args.test_profile is not None:
         pytest_args.extend(["--test-profile", _resolve_from_cwd(args.test_profile)])
     if args.html is not None:

osc_validation/pytest_plugin.py

@@ -7,7 +7,8 @@
 
 Responsibilities
 ----------------
-* Register CLI options (``--tool``, ``--toolpath``, ``--test-profile``).
+* Register CLI options (``--tool``, ``--toolpath``,
+  ``--tool-wrapper-module``, ``--test-profile``).
 * Initialise the tool under test and expose it via the ``generate_tool_trace``
   session fixture.
 * Expose optional QC OSI trace checking via the
@@ -22,8 +23,12 @@
 """
 
 import datetime
+import importlib
+import importlib.util
 import pathlib
 from urllib.parse import urlparse
+import sys
+import uuid
 
 import pytest
 
@@ -44,9 +49,17 @@ class UnknownToolError(ValueError):
     pass
 
 
+class ToolWrapperError(ValueError):
+    pass
+
+
 def _make_tool(config):
     tool_name = config.getoption("--tool")
     toolpath = config.getoption("--toolpath")
+    wrapper_module = config.getoption("--tool-wrapper-module")
+
+    if wrapper_module is not None:
+        return _make_custom_tool(wrapper_module, toolpath)
 
     if tool_name == "ESMini":
         return ESMini(toolpath)
@@ -57,12 +70,85 @@ def _make_tool(config):
     raise UnknownToolError(f"Unknown tool: {tool_name}")
 
 
+def _make_custom_tool(wrapper_module: str, toolpath: str | None):
+    module = _load_wrapper_module(wrapper_module)
+    create_tool = getattr(module, "create_tool", None)
+    if not callable(create_tool):
+        raise ToolWrapperError(
+            f"Custom tool wrapper module '{wrapper_module}' must define "
+            "a callable create_tool(toolpath)."
+        )
+
+    try:
+        tool = create_tool(toolpath)
+    except Exception as exc:
+        raise ToolWrapperError(
+            f"Custom tool wrapper module '{wrapper_module}' failed while "
+            f"creating the tool: {exc}"
+        ) from exc
+
+    if not callable(getattr(tool, "run", None)):
+        raise ToolWrapperError(
+            f"Custom tool wrapper module '{wrapper_module}' returned an object "
+            "without a callable run method."
+        )
+    return tool
+
+
+def _load_wrapper_module(wrapper_module: str):
+    module_path = pathlib.Path(wrapper_module)
+    if module_path.suffix == ".py" or module_path.exists():
+        return _load_wrapper_module_from_path(module_path, wrapper_module)
+
+    try:
+        return importlib.import_module(wrapper_module)
+    except Exception as exc:
+        raise ToolWrapperError(
+            f"Could not import custom tool wrapper module '{wrapper_module}': {exc}"
+        ) from exc
+
+
+def _load_wrapper_module_from_path(module_path: pathlib.Path, wrapper_module: str):
+    if not module_path.exists():
+        raise ToolWrapperError(
+            f"Custom tool wrapper module file does not exist: {wrapper_module}"
+        )
+    module_name = f"_osc_validation_custom_tool_{uuid.uuid4().hex}"
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    if spec is None or spec.loader is None:
+        raise ToolWrapperError(
+            f"Could not load custom tool wrapper module from file: {wrapper_module}"
+        )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    try:
+        spec.loader.exec_module(module)
+    except Exception as exc:
+        sys.modules.pop(module_name, None)
+        raise ToolWrapperError(
+            f"Could not load custom tool wrapper module from file "
+            f"'{wrapper_module}': {exc}"
+        ) from exc
+    return module
+
+
+def _get_tool_version(tool):
+    get_version = getattr(tool, "get_version", None)
+    if not callable(get_version):
+        return ["unknown version"]
+    return get_version()
+
+
 def pytest_addoption(parser):
     group = parser.getgroup("OSC Validation Suite")
     group.addoption(
         "--tool",
         action="store",
-        help="Tool to Validate: ESMini, GTGen, OscSimulator",
+        metavar="TOOL",
+        help=(
+            "Tool to validate. Built-in tools: ESMini, GTGen, OscSimulator. "
+            "Other tools require --tool-wrapper-module"
+        ),
     )
     group.addoption(
         "--toolpath",
@@ -71,6 +157,16 @@ def pytest_addoption(parser):
         metavar="PATH",
         help="Path to the tool to validate",
     )
+    group.addoption(
+        "--tool-wrapper-module",
+        action="store",
+        default=None,
+        metavar="MODULE_OR_PATH",
+        help=(
+            "Python module name or .py file path providing "
+            "create_tool(toolpath) for a custom tool wrapper"
+        ),
+    )
     group.addoption(
         "--test-profile",
         action="store",
@@ -119,10 +215,10 @@ def pytest_configure(config):
 
     try:
         tool = _make_tool(config)
-    except (UnknownToolError, FileNotFoundError) as exc:
+    except (UnknownToolError, ToolWrapperError, FileNotFoundError) as exc:
         raise pytest.UsageError(str(exc)) from exc
     config._osc_tool = tool
-    config._osc_tool_version = tool.get_version()
+    config._osc_tool_version = _get_tool_version(tool)
 
     profile_path = config.getoption("--test-profile")
     if profile_path is not None:

tests/test_cli_entrypoints.py

@@ -179,6 +179,77 @@ def fake_subprocess_run(command, check, cwd):
     assert f"--junitxml={Path('validation-results.xml').resolve()}" in captured_command
 
 
+def test_osc_validate_main_forwards_tool_wrapper_module(monkeypatch):
+    captured_command = None
+    validation_dir = Path("installed-validation")
+
+    def fake_subprocess_run(command, check, cwd):
+        nonlocal captured_command
+        captured_command = command
+        assert check is False
+        assert cwd == validation_dir
+        return type("CompletedProcess", (), {"returncode": 0})()
+
+    monkeypatch.setattr(cli_module.subprocess, "run", fake_subprocess_run)
+    monkeypatch.setattr(cli_module, "_validation_dir", lambda: validation_dir)
+    monkeypatch.setattr(
+        cli_module.Path,
+        "exists",
+        lambda self: self.name in {"pytest.ini", "scenario"},
+    )
+
+    assert (
+        cli_module.main(
+            [
+                "--tool",
+                "CustomTool",
+                "--tool-wrapper-module",
+                "custom_package.wrapper",
+            ]
+        )
+        == 0
+    )
+
+    assert "--tool" in captured_command
+    assert "CustomTool" in captured_command
+    assert "--tool-wrapper-module" in captured_command
+    assert "custom_package.wrapper" in captured_command
+
+
+def test_osc_validate_main_resolves_tool_wrapper_file(monkeypatch):
+    captured_command = None
+    validation_dir = Path("installed-validation")
+
+    def fake_subprocess_run(command, check, cwd):
+        nonlocal captured_command
+        captured_command = command
+        assert check is False
+        assert cwd == validation_dir
+        return type("CompletedProcess", (), {"returncode": 0})()
+
+    monkeypatch.setattr(cli_module.subprocess, "run", fake_subprocess_run)
+    monkeypatch.setattr(cli_module, "_validation_dir", lambda: validation_dir)
+    monkeypatch.setattr(
+        cli_module.Path,
+        "exists",
+        lambda self: self.name in {"pytest.ini", "scenario"},
+    )
+
+    assert (
+        cli_module.main(
+            [
+                "--tool",
+                "CustomTool",
+                "--tool-wrapper-module",
+                "custom_wrapper.py",
+            ]
+        )
+        == 0
+    )
+
+    assert str(Path("custom_wrapper.py").resolve()) in captured_command
+
+
 def test_osc_validate_main_rejects_positional_test_paths(monkeypatch):
     monkeypatch.setattr(
         cli_module.subprocess,