diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 81c18f19..ff9eddd6 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -30,6 +30,7 @@ import os import platform import sys +from importlib import import_module from typing import Optional import nodescraper @@ -54,12 +55,84 @@ from nodescraper.pluginexecutor import PluginExecutor from nodescraper.pluginregistry import PluginRegistry -try: - import ext_nodescraper_plugins as ext_pkg - extra_pkgs = [ext_pkg] -except ImportError: +def discover_external_plugins(): + """Discover ext_nodescraper_plugins from all installed packages. + + Returns: + list: List of discovered plugin packages + """ extra_pkgs = [] + seen_paths = set() # Track paths to avoid duplicates + + try: + import ext_nodescraper_plugins as ext_pkg + + extra_pkgs.append(ext_pkg) + if hasattr(ext_pkg, "__file__") and ext_pkg.__file__: + seen_paths.add(ext_pkg.__file__) + except ImportError: + pass + + # Discover ext_nodescraper_plugins from installed packages + try: + from importlib.metadata import distributions + + for dist in distributions(): + pkg_name = dist.metadata.get("Name", "") + if not pkg_name: + continue + + name_variants = [ + pkg_name.replace("-", "_"), + pkg_name.replace("_", "-"), + ] + + try: + top_level = dist.read_text("top_level.txt") + if top_level: + name_variants.extend(top_level.strip().split("\n")) + except Exception: + pass + + for variant in name_variants: + if not variant: + continue + + try: + module_path = f"{variant}.ext_nodescraper_plugins" + ext_pkg = import_module(module_path) + + # Check if we already have this package (by file path) + pkg_path = getattr(ext_pkg, "__file__", None) + if pkg_path and pkg_path in seen_paths: + continue + + # Add the package + extra_pkgs.append(ext_pkg) + if pkg_path: + seen_paths.add(pkg_path) + + break + + except (ImportError, AttributeError, ModuleNotFoundError): + continue + + except Exception: + pass + + return extra_pkgs + + +# Fix sys.path[0] if it's the venv/bin directory to avoid breaking editable install discovery +_original_syspath0 = sys.path[0] +if _original_syspath0.endswith("/bin") or _original_syspath0.endswith("\\Scripts"): + sys.path[0] = "" + +extra_pkgs = discover_external_plugins() + +# Restore original sys.path[0] +sys.path[0] = _original_syspath0 def build_parser( diff --git a/pyproject.toml b/pyproject.toml index 358d9cf2..831d83ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,9 +12,10 @@ classifiers = ["Topic :: Software Development"] dependencies = [ "pydantic>=2.8.2", - "paramiko~=3.5.1", + "paramiko>=3.2.0,<4.0.0", "requests", - "pytz" + "pytz", + "urllib3>=1.26.15,<2.0.0" ] [project.optional-dependencies] diff --git a/test/unit/framework/test_cli.py b/test/unit/framework/test_cli.py index cd266ed9..49cd8990 100644 --- a/test/unit/framework/test_cli.py +++ b/test/unit/framework/test_cli.py @@ -25,13 +25,21 @@ ############################################################################### import argparse import os +import sys +import tempfile +import types +from pathlib import Path +from unittest import mock import pytest from pydantic import BaseModel +from nodescraper.base import InBandDataPlugin from nodescraper.cli import cli, inputargtypes -from nodescraper.enums import SystemLocation +from nodescraper.enums import ExecutionStatus, SystemLocation +from nodescraper.interfaces import DataAnalyzer from nodescraper.models import SystemInfo +from nodescraper.pluginregistry import PluginRegistry def test_log_path_arg(): @@ -150,3 +158,220 @@ def test_system_info_builder(): ) def test_process_args(raw_arg_input, plugin_names, exp_output): assert cli.process_args(raw_arg_input, plugin_names) == exp_output + + +def test_discover_external_plugins_top_level(): + """Test discovering ext_nodescraper_plugins as a top-level import.""" + mock_ext_pkg = mock.MagicMock() + mock_ext_pkg.__file__ = "/path/to/ext_nodescraper_plugins/__init__.py" + + with mock.patch("nodescraper.cli.cli.import_module"): + with mock.patch.dict("sys.modules", {"ext_nodescraper_plugins": mock_ext_pkg}): + result = cli.discover_external_plugins() + + assert len(result) >= 1 + assert mock_ext_pkg in result + + +def test_discover_external_plugins_no_plugins(): + """Test when no external plugins are installed.""" + with mock.patch("nodescraper.cli.cli.import_module") as mock_import: + mock_import.side_effect = ImportError("No module named 'ext_nodescraper_plugins'") + + with mock.patch("importlib.metadata.distributions", return_value=[]): + result = cli.discover_external_plugins() + + assert result == [] + + +def test_discover_external_plugins_from_installed_package(): + """Test discovering plugins from installed packages (not top-level).""" + mock_dist = mock.MagicMock() + mock_dist.metadata.get.return_value = "amd-custom-package" + mock_dist.read_text.return_value = "custompackage" + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/custompackage/ext_nodescraper_plugins/__init__.py" + + def mock_import_func(module_path): + if module_path == "custompackage.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + + +def test_discover_external_plugins_deduplication(): + """Test that duplicate plugins are not added multiple times.""" + mock_ext_pkg = mock.MagicMock() + mock_ext_pkg.__file__ = "/path/to/ext_nodescraper_plugins/__init__.py" + + mock_dist1 = mock.MagicMock() + mock_dist1.metadata.get.return_value = "package-one" + mock_dist1.read_text.return_value = "package_one" + + mock_dist2 = mock.MagicMock() + mock_dist2.metadata.get.return_value = "package-two" + mock_dist2.read_text.return_value = "package_one" + + def mock_import_func(module_path): + if "package_one.ext_nodescraper_plugins" in module_path: + return mock_ext_pkg + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist1, mock_dist2]): + result = cli.discover_external_plugins() + + file_paths = [pkg.__file__ for pkg in result if hasattr(pkg, "__file__")] + assert file_paths.count(mock_ext_pkg.__file__) == 1 + + +def test_discover_external_plugins_name_variants(): + """Test that different package name variants are tried (hyphens vs underscores).""" + mock_dist = mock.MagicMock() + mock_dist.metadata.get.return_value = "amd-error-scraper" + mock_dist.read_text.side_effect = Exception("No top_level.txt") + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/amd_error_scraper/ext_nodescraper_plugins/__init__.py" + + call_count = {"count": 0} + + def mock_import_func(module_path): + call_count["count"] += 1 + if module_path == "amd_error_scraper.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + assert call_count["count"] >= 1 + + +def test_discover_external_plugins_handles_exceptions(): + """Test that discovery continues even if some packages fail.""" + mock_dist1 = mock.MagicMock() + mock_dist1.metadata.get.return_value = "good-package" + mock_dist1.read_text.return_value = "goodpackage" + + mock_dist2 = mock.MagicMock() + mock_dist2.metadata.get.side_effect = Exception("Corrupted metadata") + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/goodpackage/ext_nodescraper_plugins/__init__.py" + + def mock_import_func(module_path): + if module_path == "goodpackage.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist1, mock_dist2]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + + +def test_external_plugins_integration(): + """Integration test: Create a temporary external plugin and verify it's picked up.""" + with tempfile.TemporaryDirectory() as tmpdir: + pkg_dir = Path(tmpdir) / "test_external_pkg" + pkg_dir.mkdir() + (pkg_dir / "__init__.py").write_text("") + + ext_plugins_dir = pkg_dir / "ext_nodescraper_plugins" + ext_plugins_dir.mkdir() + (ext_plugins_dir / "__init__.py").write_text("") + + plugin_module_dir = ext_plugins_dir / "test_plugin" + plugin_module_dir.mkdir() + + plugin_code = """ +from nodescraper.base import InBandDataPlugin +from nodescraper.enums import ExecutionStatus +from nodescraper.interfaces import DataAnalyzer + +class TestAnalyzer(DataAnalyzer): + DATA_MODEL = dict + + def analyze_data(self, data): + return ExecutionStatus.SUCCESS, None + +class TestExternalPlugin(InBandDataPlugin): + DATA_MODEL = dict + ANALYZER = TestAnalyzer + + def run(self): + return ExecutionStatus.SUCCESS, {"test": "data"} +""" + (plugin_module_dir / "__init__.py").write_text(plugin_code) + + sys.path.insert(0, tmpdir) + + try: + import test_external_pkg.ext_nodescraper_plugins as test_ext_pkg + + plugin_registry = PluginRegistry(plugin_pkg=[test_ext_pkg]) + + assert ( + "TestExternalPlugin" in plugin_registry.plugins + ), f"External plugin not found. Available plugins: {list(plugin_registry.plugins.keys())}" + + plugin_class = plugin_registry.plugins["TestExternalPlugin"] + assert plugin_class.__name__ == "TestExternalPlugin" + + finally: + sys.path.remove(tmpdir) + modules_to_remove = [ + key for key in sys.modules.keys() if key.startswith("test_external_pkg") + ] + for module in modules_to_remove: + del sys.modules[module] + + +def test_discover_and_load_external_plugins(): + """Test the full flow: discover external plugins using mocked modules.""" + mock_plugin_module = types.ModuleType("mock_ext_nodescraper_plugins") + mock_plugin_module.__file__ = "/fake/path/mock_ext_nodescraper_plugins/__init__.py" + mock_plugin_module.__path__ = ["/fake/path/mock_ext_nodescraper_plugins"] + + mock_submodule = types.ModuleType("mock_ext_nodescraper_plugins.mock_plugin") + mock_submodule.__file__ = "/fake/path/mock_ext_nodescraper_plugins/mock_plugin.py" + + class MockAnalyzer(DataAnalyzer): + DATA_MODEL = dict + + def analyze_data(self, data): + return ExecutionStatus.SUCCESS, None + + class MockExternalPlugin(InBandDataPlugin): + DATA_MODEL = dict + ANALYZER = MockAnalyzer + + def run(self): + return ExecutionStatus.SUCCESS, {} + + mock_submodule.MockExternalPlugin = MockExternalPlugin + + def mock_iter_modules(path, prefix=""): + yield None, f"{prefix}mock_plugin", False + + def mock_import_module(name): + if "mock_plugin" in name: + return mock_submodule + raise ImportError(f"No module named {name}") + + with mock.patch("pkgutil.iter_modules", side_effect=mock_iter_modules): + with mock.patch("importlib.import_module", side_effect=mock_import_module): + plugin_registry = PluginRegistry(plugin_pkg=[mock_plugin_module]) + + assert "MockExternalPlugin" in plugin_registry.plugins + assert plugin_registry.plugins["MockExternalPlugin"] == MockExternalPlugin