diff --git a/FEATURES.ini b/FEATURES.ini new file mode 100644 index 000000000..709a8f530 --- /dev/null +++ b/FEATURES.ini @@ -0,0 +1,8 @@ +[CLOUD] + +[MIGUX] +default_on = True +has_postinstall = True +feature_url = http://foo.bar/baz + +[WORKFLOWS] diff --git a/local-requirements.txt b/local-requirements.txt index 4941e1a6b..a66af4662 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -9,4 +9,5 @@ isort # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support paramiko;python_version >= "3.7" paramiko<3;python_version < "3.7" +python-dotenv werkzeug diff --git a/mig/install/features.py b/mig/install/features.py new file mode 100755 index 000000000..b67c3fcc6 --- /dev/null +++ b/mig/install/features.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# features - bootstrap tool for managing per-feature dependencies +# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +import argparse +import importlib +import os +import sys +from collections import defaultdict +from configparser import ConfigParser +from enum import Enum +from types import SimpleNamespace + +import pip +from pip._internal.req.req_file import parse_requirements + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_LOCAL_MIG_BASE = os.path.normpath(os.path.join(_SCRIPT_DIR, "../..")) + +sys.path.append(_LOCAL_MIG_BASE) + +FEATURES_FILE = os.path.join(_LOCAL_MIG_BASE, "FEATURES.ini") +FEATURES_REQUIREMENTS_DIR = os.path.join( + _LOCAL_MIG_BASE, "mig/install/requirements" +) +PIP_OVERRIDES = { + "CLOUD": { + "openstacksdk": "OPENSTACKSDK_VERSION_OVERRIDE", + }, + "MIGUX": { + "migux": "MIGUX_VERSION_OVERRIDE", + }, +} +_VERSIONCHARS = ("=", "<", ">") +_TRUTH_STRINGS = set(("True", "true", "yes", "1")) + + +def warn(msg=""): + """ + Wrapper function for printing to stderr. + """ + print(msg, file=sys.stderr) + + +class Features: + """ + Instances of this object represent a set of named features and their state. + """ + + def __init__(self, interpretation_by_feature_name, overrides_supported): + self.feature_names = sorted(interpretation_by_feature_name.keys()) + self._enabled_by_feature = {} + self._requirements_by_feature = {} + self._requirements_file_by_feature = {} + self._overrides_by_feature = {} + self._overrides_supported = overrides_supported + + for ( + feature_name, + interpretation, + ) in interpretation_by_feature_name.items(): + self._enabled_by_feature[feature_name] = interpretation.enabled + self._requirements_by_feature[feature_name] = ( + interpretation.requirements + ) + self._requirements_file_by_feature[feature_name] = ( + interpretation.requirements_file + ) + + def apply_enabled(self, enabled_by_feature_name): + """ + Update the enabled state of features. + """ + + feature_keys = set(self.feature_names) + present_keys = set(enabled_by_feature_name.keys()) + + missing_feature_keys = feature_keys - present_keys + if missing_feature_keys: + raise RuntimeError("supplied feature state incomplete") + + self._enabled_by_feature = enabled_by_feature_name + + def apply_overrides(self, overrides_by_feature_name): + """ + Update the overrides associated with features. + """ + + self._overrides_by_feature = overrides_by_feature_name + + def feature_is_enabled(self, feature_name): + """ + Check if a named feature is enabled. + """ + + return self._enabled_by_feature[feature_name] + + def generate_pip_args(self, detect_installed=False): + """ + Create pip arguments for each enabled feature. + """ + + per_package_args = [] + + for feature_name in self.list_enabled_features(): + installable_packages = self.generate_pip_args_for_feature( + feature_name, detect_installed=detect_installed + ) + if not installable_packages: + continue + per_package_args.append(installable_packages) + + return per_package_args + + def generate_pip_args_for_feature(self, feature_name, *, detect_installed): + """ + Create pip arguments for a particular feature. + """ + + overrides = self._overrides_by_feature.get(feature_name, {}) + + if not (overrides or detect_installed): + # no overrides detected and we do not need to check for the + # dependencies being already installed therefore we can install + # by simply using the requirements file as-is + return ["-r", self._requirements_file_by_feature[feature_name]] + + package_args = [] + overridden_package_names = set(overrides.keys()) + + # add the overridden packages + for package_name in overridden_package_names: + package_args.append(f"{package_name}=={overrides[package_name]}") + + if detect_installed: + packages_to_detect = self.required_package_names(feature_name) + else: + packages_to_detect = set() + + # add the remaining packages based on the requirements file + for entry in self._requirements_by_feature[feature_name]: + package_name = Features._strip_version_if_present(entry.requirement) + if package_name in overridden_package_names: + continue + + should_detect = package_name in packages_to_detect + if should_detect: + skip_installation = Features._is_package_present(package_name) + else: + skip_installation = False + + if skip_installation: + continue + package_args.append(entry.requirement) + + return package_args + + def list_enabled_features(self, return_as=list): + """ + Return the names of features recorded as enabled. + """ + + return return_as( + ( + feature_name + for feature_name in self.feature_names + if self._enabled_by_feature[feature_name] + ) + ) + + def required_package_names(self, feature_name): + """ + Return the set of required packages for a named feature. + """ + return set( + ( + Features._strip_version_if_present(entry.requirement) + for entry in self._requirements_by_feature[feature_name] + ) + ) + + @staticmethod + def _interpret_feature_definition( + feature_name, feature_definition, requirements_dir + ): + """ + Convert a named feature section within the features file to a + structured intepretation suitable for consumption by the logic. + """ + + enabled = feature_definition.getboolean("default_on", fallback=False) + has_requirements = feature_definition.getboolean( + "has_requirements", fallback=True + ) + + if has_requirements: + requirements_file = os.path.join( + requirements_dir, f"{feature_name.lower()}-requirements.txt" + ) + requirements = list( + parse_requirements(requirements_file, session=None) + ) + else: + requirements = [] + + return SimpleNamespace( + enabled=enabled, + requirements=requirements, + requirements_file=requirements_file, + ) + + @staticmethod + def _is_package_present(package_name): + """ + Determine whether a package is available to the active interpreter. + """ + + try: + importlib.util.find_spec(package_name) + return True + except ModuleNotFoundError as exc: + return False + + @staticmethod + def _strip_version_if_present(requirement): + """ + Return only the name of a package given a requirement specifier. + """ + + for char in _VERSIONCHARS: + index = requirement.find(char) + if index == -1: + continue + return requirement[:index] + return requirement + + @staticmethod + def expand_definitions(definitions, requirements_dir): + """ + Generate a dictionary of features names and a structured interpretation + based on their definition in the main features file. + """ + + definitions_iterator = iter(definitions.items()) + next(definitions_iterator) # skip default section + + return { + feature_name: Features._interpret_feature_definition( + feature_name, feature_definition, requirements_dir + ) + for feature_name, feature_definition in definitions_iterator + } + + @classmethod + def from_definitions_file( + cls, features_file, requirements_dir, overrides_supported={} + ): + """ + Return a Features instance populated with the features declared + within the specified definitions file. + """ + + assert os.path.isabs(features_file) + with open(features_file) as thefile: + definitions = ConfigParser() + definitions.read_file(thefile) + return cls( + Features.expand_definitions(definitions, requirements_dir), + overrides_supported, + ) + + @staticmethod + def match_env_dict(features, env_dict): + """ + Check the supplied dictionary for env-style flags (i.e. ENABLE_) + which indicate whether the correspnding feature should be enabled and + optionally - based upon the definitions - for any applicable version + overrides being specified if such flags are supported. + """ + + def enabled_or_fallback(feature_name): + try: + enable_string = env_dict[f"ENABLE_{feature_name.upper()}"] + return enable_string in _TRUTH_STRINGS + except KeyError: + return features.feature_is_enabled(feature_name) + + enabled_by_feature_name = {} + overrides_by_feature_name = defaultdict(dict) + + for feature_name in features.feature_names: + enabled_by_feature_name[feature_name] = enabled_or_fallback( + feature_name + ) + + env_override_flags = features._overrides_supported.get( + feature_name, None + ) + if not env_override_flags: + continue + + for package_name, flag_name in env_override_flags.items(): + override_version = env_dict.get(flag_name, None) + if not override_version: + continue + overrides_by_feature_name[feature_name][ + package_name + ] = override_version + + return enabled_by_feature_name, overrides_by_feature_name + + @staticmethod + def match_dotenv_file(features, dotenv_file): + """ + Match a .env file for feature enablement and overrides. + """ + + from dotenv import dotenv_values + + assert os.path.isabs(dotenv_file) + dotenv_dict = dotenv_values(dotenv_file) + + return Features.match_env_dict(features, dotenv_dict) + + @staticmethod + def match_configuration_file(features, configuration_file): + """ + Match a .configuration file for feature enablement and overrides. + """ + + from mig.shared.conf import get_configuration_object + + configuration = get_configuration_object( + configuration_file, skip_log=True, disable_auth_log=True + ) + + def enabled_or_fallback(feature_name): + try: + return getattr( + configuration, f"site_enable_{feature_name.lower()}" + ) + except AttributeError: + return features.feature_is_enabled(feature_name) + + enabled_by_feature_name = { + feature_name: enabled_or_fallback(feature_name) + for feature_name in features.feature_names + } + return enabled_by_feature_name, {} + + +def subcommand_enabled(features, args, print=print, warn=warn): + """ + 'enabled' subcommand executor. + """ + + if args.c: + enabled_by_feature_name = Features.match_configuration_file( + features, args.c + ) + features.apply_enabled(enabled_by_feature_name) + elif args.dotenv: + enabled_by_feature_name, _ = Features.match_dotenv_file( + features, args.dotenv + ) + features.apply_enabled(enabled_by_feature_name) + elif args.env: + enabled_by_feature_name, overrides_by_feature_name = ( + Features.match_env_dict(features, args.env) + ) + features.apply_enabled(enabled_by_feature_name) + else: + warn( + "no feature coniguration available; showing those enabled by default only" + ) + print(f"enabled features: {', '.join(features.list_enabled_features())}") + + return 0 + + +def subcommand_install(features, args, print=print, warn=warn): + """ + 'install' subcommand executor. + """ + + if args.c: + enabled_by_feature_name = Features.match_configuration_file( + features, args.c + ) + features.apply_enabled(enabled_by_feature_name) + elif args.dotenv: + enabled_by_feature_name, overrides_by_feature_name = ( + Features.match_dotenv_file(features, args.dotenv) + ) + features.apply_enabled(enabled_by_feature_name) + elif args.env: + enabled_by_feature_name, overrides_by_feature_name = ( + Features.match_env_dict(features, args.env) + ) + features.apply_enabled(enabled_by_feature_name) + features.apply_overrides(overrides_by_feature_name) + else: + warn( + "no feature coniguration available; showing those enabled by default only" + ) + warn() + + all_pip_args = features.generate_pip_args( + detect_installed=args.detect_installed + ) + + if args.check: + for pip_args in all_pip_args: + print(f"pip install {' '.join(pip_args)}") + return + + raise NotImplementedError("install is not currently implemented") + + +def subcommand_show(features, args, print=print, warn=warn): + """ + 'show' subcommand executor. + """ + + print(f"available features: {', '.join(features.feature_names)}") + + +_COMMAND_HANDLERS = dict( + enabled=subcommand_enabled, + install=subcommand_install, + show=subcommand_show, +) + + +def main(argv): + """ + Main entrypoint function. + """ + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + + show_command = subparsers.add_parser("show") + + enabled_command = subparsers.add_parser("enabled") + enabled_command.add_argument("-c", default=None) + enabled_command.add_argument("--dotenv", default=None, type=os.path.abspath) + enabled_command.add_argument( + "--env", action="store_const", const=os.environ + ) + + install_command = subparsers.add_parser("install") + install_command.add_argument("-c", default=None) + install_command.add_argument("--check", action="store_true", default=False) + install_command.add_argument( + "--detect_installed", action="store_true", default=False + ) + install_command.add_argument("--dotenv", default=None, type=os.path.abspath) + install_command.add_argument( + "--env", action="store_const", const=os.environ + ) + + args = parser.parse_args(args=argv) + + if not args.command: + parser.print_usage() + return 0 + + return args_main(parser.parse_args(args=argv)) + + +def args_main(args, *, print=print, warn=warn, features=None): + """ + Internal helper for executing the parsed arguments. + """ + + features = features or Features.from_definitions_file( + FEATURES_FILE, + FEATURES_REQUIREMENTS_DIR, + PIP_OVERRIDES, + ) + + command_handler = _COMMAND_HANDLERS[args.command] + try: + command_handler(features, args, print=print, warn=warn) + return 0 + except Exception as exc: + warn(exc) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/mig/install/requirements/cloud-requirements.txt b/mig/install/requirements/cloud-requirements.txt new file mode 100644 index 000000000..5852d7b8a --- /dev/null +++ b/mig/install/requirements/cloud-requirements.txt @@ -0,0 +1 @@ +openstacksdk==4.5.6 diff --git a/mig/install/requirements/migux-requirements.txt b/mig/install/requirements/migux-requirements.txt new file mode 100644 index 000000000..db0918341 --- /dev/null +++ b/mig/install/requirements/migux-requirements.txt @@ -0,0 +1 @@ +migux diff --git a/mig/install/requirements/workflows-requirements.txt b/mig/install/requirements/workflows-requirements.txt new file mode 100644 index 000000000..74cf1b53d --- /dev/null +++ b/mig/install/requirements/workflows-requirements.txt @@ -0,0 +1,3 @@ +nbconvert +nbformat +papermill diff --git a/requirements-feature-migux.txt b/requirements-feature-migux.txt new file mode 100644 index 000000000..db0918341 --- /dev/null +++ b/requirements-feature-migux.txt @@ -0,0 +1 @@ +migux diff --git a/tests/data/features/basic/.env--enable-foo b/tests/data/features/basic/.env--enable-foo new file mode 100644 index 000000000..ddcf2520b --- /dev/null +++ b/tests/data/features/basic/.env--enable-foo @@ -0,0 +1 @@ +ENABLE_FOO = True diff --git a/tests/data/features/basic/features.ini b/tests/data/features/basic/features.ini new file mode 100644 index 000000000..67bcc5409 --- /dev/null +++ b/tests/data/features/basic/features.ini @@ -0,0 +1,6 @@ +[FOO] + +[BAR] + +[BAZ] +default_on = True diff --git a/tests/data/features/basic/requirements/bar-requirements.txt b/tests/data/features/basic/requirements/bar-requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/features/basic/requirements/baz-requirements.txt b/tests/data/features/basic/requirements/baz-requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/features/basic/requirements/foo-requirements.txt b/tests/data/features/basic/requirements/foo-requirements.txt new file mode 100644 index 000000000..6bd87cdad --- /dev/null +++ b/tests/data/features/basic/requirements/foo-requirements.txt @@ -0,0 +1,2 @@ +somepkg==4.5.6 +otherpkg==4.5.6 diff --git a/tests/data/features/detection/features.ini b/tests/data/features/detection/features.ini new file mode 100644 index 000000000..71cfc50bb --- /dev/null +++ b/tests/data/features/detection/features.ini @@ -0,0 +1 @@ +[EXISTS] diff --git a/tests/data/features/detection/requirements/exists-requirements.txt b/tests/data/features/detection/requirements/exists-requirements.txt new file mode 100644 index 000000000..a31743a27 --- /dev/null +++ b/tests/data/features/detection/requirements/exists-requirements.txt @@ -0,0 +1,3 @@ +# deliberately choose a package that is always installed in development +# such that requesting detection will find it and skip its installation +autopep8 diff --git a/tests/test_mig_install_features.py b/tests/test_mig_install_features.py new file mode 100644 index 000000000..a70138e48 --- /dev/null +++ b/tests/test_mig_install_features.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_install_generateconfs - unit test of the corresponding mig module +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the migrid module pointed to in the filename""" + +from __future__ import print_function + +import importlib +import os +import sys +from types import SimpleNamespace + +from tests.support import TEST_DATA_DIR, MigTestCase, testmain + + +from mig.install.features import args_main, Features + + +TEST_FEATURES_EXAMPLES_DIR = os.path.join(TEST_DATA_DIR, 'features') + + +class FakePrint: + def __init__(self): + self._lines = [] + + def __call__(self, value=''): + self._lines.append(value) + + +def _make_example_features_instance(example_name, overrides_supported={}): + example_dir = os.path.join(TEST_FEATURES_EXAMPLES_DIR, example_name) + features_file = os.path.join(example_dir, 'features.ini') + requirements_dir = os.path.join(example_dir, 'requirements') + + features = Features.from_definitions_file(features_file, + requirements_dir, + overrides_supported) + + return example_dir, features + + +class MigInstallFeatures_logic(MigTestCase): + """Unit test helper for the migrid code pointed to in class name""" + + def assertOutputLines(self, fake_print, expected_lines): + assert isinstance(fake_print, FakePrint) + + self.assertEqual(fake_print._lines, expected_lines) + + def test_command_show(self): + args = SimpleNamespace( + command='show', + ) + fake_print = FakePrint() + _, features = _make_example_features_instance('basic') + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + "available features: BAR, BAZ, FOO", + ]) + + def test_command_enabled_default_on(self): + args = SimpleNamespace( + command='enabled', + c=None, + dotenv=None, + env=None, + ) + fake_print = FakePrint() + fake_warn = FakePrint() + _, features = _make_example_features_instance('basic') + + ret = args_main(args, print=fake_print, warn=fake_warn, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_warn, [ + "no feature coniguration available; showing those enabled by default only" + ]) + self.assertOutputLines(fake_print, [ + "enabled features: BAZ", + ]) + + def test_command_enabled_using_dotenv(self): + fake_print = FakePrint() + example_dir, features = _make_example_features_instance('basic') + args = SimpleNamespace( + command='enabled', + c=None, + dotenv=os.path.join(example_dir, '.env--enable-foo'), + env=None, + ) + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + "enabled features: BAZ, FOO", + ]) + + def test_command_enabled_using_env(self): + fake_print = FakePrint() + example_dir, features = _make_example_features_instance('basic') + args = SimpleNamespace( + command='enabled', + c=None, + dotenv=None, + env={ + 'ENABLE_BAR': 'true', + }, + ) + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + "enabled features: BAR, BAZ", + ]) + + def test_command_install_check(self): + fake_print = FakePrint() + example_dir, features = _make_example_features_instance('basic') + args = SimpleNamespace( + command='install', + check=True, + detect_installed=False, + c=None, + dotenv=os.path.join(example_dir, '.env--enable-foo'), + env=None + ) + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + f"pip install -r {os.path.join(example_dir, 'requirements/baz-requirements.txt')}", + f"pip install -r {os.path.join(example_dir, 'requirements/foo-requirements.txt')}", + ]) + + def test_command_install_conflicting_package_versions(self): + fake_print = FakePrint() + example_dir, features = _make_example_features_instance('detection') + args = SimpleNamespace( + command='install', + check=True, + detect_installed=True, + c=None, + dotenv=None, + env={ + 'ENABLE_EXISTS': 'true' + } + ) + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, []) + + def test_command_install_overridden_package_version(self): + fake_print = FakePrint() + example_dir, features = _make_example_features_instance('basic', + overrides_supported={ + 'FOO': { + 'somepkg': 'OVERRIDE_SOMEPKG_VERSION', + } + }) + args = SimpleNamespace( + command='install', + check=True, + detect_installed=False, + c=None, + dotenv=None, + env={ + 'ENABLE_FOO': 'true', + 'OVERRIDE_SOMEPKG_VERSION': '4.5.5', + } + ) + + ret = args_main(args, print=fake_print, features=features) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + f"pip install -r {os.path.join(example_dir, 'requirements/baz-requirements.txt')}", + "pip install somepkg==4.5.5 otherpkg==4.5.6", + ]) + + +class MigInstallFeatures_smoke(MigTestCase): + """Unit test helper for the migrid code pointed to in class name""" + + def assertOutputLines(self, fake_print, expected_lines): + assert isinstance(fake_print, FakePrint) + + self.assertEqual(fake_print._lines, expected_lines) + + def test_command_show(self): + args = SimpleNamespace( + command='show', + ) + fake_print = FakePrint() + + ret = args_main(args, print=fake_print) + + self.assertEqual(ret, 0) + self.assertOutputLines(fake_print, [ + "available features: CLOUD, MIGUX, WORKFLOWS", + ]) + + +if __name__ == '__main__': + testmain()