From 3377debb13382ce7204d52fde1fd64ce382566bf Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Mon, 14 Sep 2026 18:02:27 -0700 Subject: [PATCH] Implement lazy directives for Ramble Convert eager directive execution into on-demand lazy evaluation via non-data descriptors, eliminating upfront convert_class_attributes deepcopies and isolating instance mutations. Signed-off-by: Lin Guo --- conftest.py | 2 +- lib/ramble/ramble/cmd/common/info.py | 2 +- .../ramble/language/application_language.py | 26 +- lib/ramble/ramble/language/language_base.py | 596 +++++++++------ .../ramble/language/modifier_language.py | 23 +- .../language/package_manager_language.py | 19 +- .../ramble/language/platform_language.py | 13 +- lib/ramble/ramble/language/shared_language.py | 51 +- lib/ramble/ramble/language/system_language.py | 25 +- .../ramble/language/utility_language.py | 10 +- .../language/workflow_manager_language.py | 19 +- .../test/language/test_lazy_directives.py | 714 ++++++++++++++++++ .../test/language/test_requires_utility.py | 7 - lib/ramble/ramble/test/mirror.py | 69 +- lib/ramble/ramble/util/class_attributes.py | 28 - lib/ramble/ramble/util/directives.py | 23 +- .../application-base/base_class.py | 31 +- .../base_classes/modifier-base/base_class.py | 15 +- .../base_classes/object-mixin/base_class.py | 46 +- .../package-manager-base/base_class.py | 13 +- .../base_classes/platform-base/base_class.py | 11 +- .../base_classes/system-base/base_class.py | 16 +- .../base_classes/utility-base/base_class.py | 23 +- .../workflow-manager-base/base_class.py | 11 +- 24 files changed, 1318 insertions(+), 475 deletions(-) create mode 100644 lib/ramble/ramble/test/language/test_lazy_directives.py delete mode 100644 lib/ramble/ramble/util/class_attributes.py diff --git a/conftest.py b/conftest.py index e31510cdb7..34db0443fa 100644 --- a/conftest.py +++ b/conftest.py @@ -553,7 +553,7 @@ def clear_directive_functions(): # functions. import ramble.language.language_base - ramble.language.language_base.DirectiveMeta._directives_to_be_executed = [] + ramble.language.language_base.DirectiveMeta._directives_to_be_executed.clear() @pytest.fixture diff --git a/lib/ramble/ramble/cmd/common/info.py b/lib/ramble/ramble/cmd/common/info.py index c5038ed5e1..e99aa6a641 100644 --- a/lib/ramble/ramble/cmd/common/info.py +++ b/lib/ramble/ramble/cmd/common/info.py @@ -53,7 +53,7 @@ "registered_phases": "phase_definitions", # Modifier specific: "modes": None, - "default_mode": "_default_usage_mode", + "default_mode": "default_usage_mode", "variable_modifications": None, "executable_modifiers": None, "env_var_modifications": None, diff --git a/lib/ramble/ramble/language/application_language.py b/lib/ramble/ramble/language/application_language.py index e7ea519495..46a4222b08 100644 --- a/lib/ramble/ramble/language/application_language.py +++ b/lib/ramble/ramble/language/application_language.py @@ -6,6 +6,8 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools + import ramble.definitions.variables import ramble.language.language_helpers import ramble.language.shared_language @@ -42,12 +44,8 @@ class Gromacs(ExecutableApplication): """ -class ApplicationMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - - -application_directive = ApplicationMeta.directive +ApplicationMeta = ramble.language.shared_language.SharedMeta +application_directive = functools.partial(ApplicationMeta.directive, language_type="application") @application_directive("workloads") @@ -98,7 +96,9 @@ def _execute_workload(app): return _execute_workload -@application_directive("workload_groups") +@application_directive( + dicts=("workload_groups", "workload_group_vars", "workload_group_env_vars", "workloads") +) def workload_group(name, workloads=None, mode=None, when=None, **kwargs): """Adds a workload group to this application @@ -246,7 +246,15 @@ def _execute_input_file(app): return _execute_input_file -@application_directive("workload_group_vars") +@application_directive( + dicts=( + "workload_group_vars", + "workload_group_env_vars", + "workload_groups", + "workloads", + "validators", + ) +) def workload_variable( name, default=None, @@ -431,7 +439,7 @@ def _execute_workload_variable(app): return _execute_workload_variable -@application_directive(dicts=()) +@application_directive(dicts="license_names", init_value=[]) def license_name(name, **kwargs): """Add a new license name directive, to specify license name in a declarative way. diff --git a/lib/ramble/ramble/language/language_base.py b/lib/ramble/ramble/language/language_base.py index e37e228df6..d34eaf0025 100644 --- a/lib/ramble/ramble/language/language_base.py +++ b/lib/ramble/ramble/language/language_base.py @@ -11,20 +11,18 @@ """ import abc +import collections import copy import functools import inspect -from collections.abc import Sequence # novm -from typing import Any, Callable, Dict, List, Set - -import llnl.util.lang +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Type, Union import ramble.language.language_helpers from ramble.error import DirectiveError from ramble.util import directives from ramble.util.logger import logger -__all__ = ["DirectiveMeta", "DirectiveError"] +__all__ = ["DirectiveMeta", "DirectiveDictDescriptor", "DirectiveError"] def _impossible_when_warning(directive_name, obj_type, obj_name, message, args, kwargs): @@ -51,44 +49,74 @@ def _impossible_when_warning(directive_name, obj_type, obj_name, message, args, #: them reserved_names: List[str] = [] -namespaces = [ - "ramble.app", - "ramble.mod", - "ramble.pkg_man", - "ramble.package_manager", - "ramble.wm", - "ramble.workflow_manager", - "ramble.sys", - "ramble.system", - "ramble.plat", - "ramble.platform", - "ramble.base_cls", - "ramble.modifier", - "ramble.ext_dep", - "ramble.utility", -] - - -def _push_to_context(when_condition: str) -> None: - DirectiveMeta._when_constraints_from_context.append(when_condition) - - impossible, message = ramble.language.language_helpers.is_when_impossible( - DirectiveMeta._when_constraints_from_context - ) - if impossible: - logger.warn(f"Entering an impossible 'when' context: {message}") - - -def _pop_from_context() -> str: - return DirectiveMeta._when_constraints_from_context.pop() - +_UNSET = object() -def _push_default_args(default_args: Dict[str, Any]) -> None: - DirectiveMeta._default_args.append(default_args) +def _copy_directive_value(val: Any) -> Any: + """Fast copy helper for directive values. -def _pop_default_args() -> dict: - return DirectiveMeta._default_args.pop() + Returns primitives and immutables directly, allocates new empty containers + for empty collections, and deepcopies populated collections for isolation. + """ + if val is None or isinstance(val, (int, float, str, bool, tuple, frozenset)): + return val + if not val: + return type(val)() + return copy.deepcopy(val) + + +class DirectiveDictDescriptor: + """A descriptor that lazily executes directives on first access.""" + + def __init__(self, name: str) -> None: + self.name = name + self.private_name = f"_{name}" + + def _evaluate_class(self, cls: type) -> Any: + """Lazily evaluate directives on the class if not already done.""" + val = cls.__dict__.get(self.private_name, _UNSET) + if val is not _UNSET: + return val + + dicts_to_init, directives_to_run = DirectiveMeta.get_cached_execution_plan(self.name) + class_values = getattr(cls, "_class_directive_values", {}) + initialized_dicts = [] + for dictionary in dicts_to_init: + if cls.__dict__.get(f"_{dictionary}", _UNSET) is _UNSET: + if dictionary in class_values: + init_val = class_values[dictionary] + else: + init_val = DirectiveMeta._directive_init_values.get(dictionary, {}) + setattr(cls, f"_{dictionary}", _copy_directive_value(init_val)) + initialized_dicts.append(dictionary) + + directives_list = getattr(cls, "_directives_to_be_executed", []) + DirectiveMeta._executing_directives_depth += 1 + try: + for directive_name, directive in directives_list: + if directive_name in directives_to_run: + directive(cls) + except Exception: + for dictionary in initialized_dicts: + setattr(cls, f"_{dictionary}", _UNSET) + raise + finally: + DirectiveMeta._executing_directives_depth -= 1 + + res = cls.__dict__.get(self.private_name, _UNSET) + return res if res is not _UNSET else None + + def __get__(self, obj: Any, objtype: Optional[type] = None) -> Any: + if obj is None: + if objtype is None: + return self + return self._evaluate_class(objtype) + + target_cls = objtype if objtype is not None else type(obj) + cls_val = self._evaluate_class(target_cls) + inst_val = _copy_directive_value(cls_val) + obj.__dict__[self.name] = inst_val + return inst_val class DirectiveMeta(abc.ABCMeta): @@ -96,206 +124,325 @@ class DirectiveMeta(abc.ABCMeta): area into the package. """ - # Set of all known directives - _directive_names: Set[str] = set() + # Depth counter indicating whether directives are actively being executed + _executing_directives_depth: int = 0 + + # Registry mapping directive_name -> Tuple[dict_name, ...] + _directive_to_dicts: Dict[str, Tuple[str, ...]] = {} + _dict_to_directives: Dict[str, List[str]] = collections.defaultdict(list) + # Cache of DirectiveDictDescriptor instances + _descriptor_cache: Dict[str, DirectiveDictDescriptor] = {} + # Cache of execution plans: dict_name -> (dicts_to_init, set_of_directives_to_run) + _execution_plan_cache: Dict[str, Tuple[List[str], Set[str]]] = {} + # Set of all known directive dictionary names + _directive_dict_names: Set[str] = set() + # Map of dict_name -> initial value template _directive_init_values: Dict[str, Any] = {} - _directives_to_be_executed: List[Callable[..., Any]] = [] + # List of directives to be executed for the class being defined, preserving definition order + _directives_to_be_executed: List[Tuple[str, Any]] = [] + # Directive functions and classes _directive_functions: Dict[str, Callable[..., Any]] = {} _directive_classes: Dict[str, type] = {} + _directive_types: Dict[str, str] = {} + # Workload-related dictionaries referenced by environment_variable in shared_language + # that belong exclusively to applications + _cross_boundary_app_dicts: Set[str] = { + "workloads", + "workload_groups", + "workload_group_vars", + "workload_group_env_vars", + } + # Dynamic mapping of object types to their exclusive directive dictionaries + _type_scoped_dicts: Dict[str, Set[str]] = collections.defaultdict(set) + # Set of dictionaries registered by shared directives + _shared_dict_names: Set[str] = set() _when_constraints_from_context: List[str] = [] _default_args: List[dict] = [] - push_to_context = _push_to_context - pop_from_context = _pop_from_context - push_default_args = _push_default_args - pop_default_args = _pop_default_args - - def __new__(cls, name, bases, attr_dict): - # Initialize the attribute containing the list of directives - # to be executed. Here we go reversed because we want to execute - # commands: - # 1. in the order they were defined - # 2. following the MRO - attr_dict["_directives_to_be_executed"] = [] - for base in reversed(bases): - directive_from_base = getattr(base, "_directives_to_be_executed", None) - if directive_from_base is not None: - attr_dict["_directives_to_be_executed"].extend(directive_from_base) - - # De-duplicates directives from base classes - attr_dict["_directives_to_be_executed"] = list( - llnl.util.lang.dedupe(attr_dict["_directives_to_be_executed"]) + @staticmethod + def push_to_context(when_condition: str) -> None: + """Push a when condition onto the context stack.""" + DirectiveMeta._when_constraints_from_context.append(when_condition) + impossible, message = ramble.language.language_helpers.is_when_impossible( + DirectiveMeta._when_constraints_from_context ) + if impossible: + logger.warn(f"Entering an impossible 'when' context: {message}") - # Move things to be executed from module scope (where they - # are collected first) to class scope - if DirectiveMeta._directives_to_be_executed: - attr_dict["_directives_to_be_executed"].extend( - DirectiveMeta._directives_to_be_executed - ) - DirectiveMeta._directives_to_be_executed = [] - - return super().__new__(cls, name, bases, attr_dict) + @staticmethod + def pop_from_context() -> str: + """Pop the last when condition from the context stack.""" + return DirectiveMeta._when_constraints_from_context.pop() - def __init__(cls, name, bases, attr_dict): - # The instance is being initialized: if it is a package we must ensure - # that the directives are called to set it up. - - valid_module = False - for namespace in namespaces: - if namespace in cls.__module__: - valid_module = True - - if valid_module: - # Ensure the presence of the dictionaries associated - # with the directives - # We use type(cls) to get the metaclass, and iterate its MRO to - # collect all init values and directive attributes. - all_init_values = {} - all_directive_names = set() - all_directive_functions = {} - all_directive_classes = {} - - for base_meta in reversed(type(cls).__mro__): - if hasattr(base_meta, "_directive_init_values"): - all_init_values.update(base_meta._directive_init_values) - if hasattr(base_meta, "_directive_names"): - all_directive_names |= base_meta._directive_names - if hasattr(base_meta, "_directive_functions"): - all_directive_functions.update(base_meta._directive_functions) - if hasattr(base_meta, "_directive_classes"): - all_directive_classes.update(base_meta._directive_classes) - - for d, t in all_init_values.items(): - setattr(cls, d, copy.deepcopy(t)) - - directive_attrs = { - "_directive_functions": all_directive_functions, - "_directive_classes": all_directive_classes, - "_directive_names": all_directive_names | DirectiveMeta._directive_names.copy(), - } - - for attr, val in directive_attrs.items(): - if hasattr(DirectiveMeta, attr): - val.update(getattr(DirectiveMeta, attr)) - - for attr, val in directive_attrs.items(): - setattr(cls, attr, val) - - # Lazily execute directives - for directive in cls._directives_to_be_executed: - directive(cls) - - # Ignore any directives executed *within* top-level - # directives by clearing out the queue they're appended to - DirectiveMeta._directives_to_be_executed = [] - - directives.define_directive_methods_on_class(cls) + @staticmethod + def push_default_args(default_args: Dict[str, Any]) -> None: + """Push default arguments onto the stack.""" + DirectiveMeta._default_args.append(default_args) - super().__init__(name, bases, attr_dict) + @staticmethod + def pop_default_args() -> dict: + """Pop default arguments from the stack.""" + return DirectiveMeta._default_args.pop() @classmethod - def directive(cls, dicts=None, init_value=None): - """Decorator for Ramble directives. - - Ramble directives allow you to modify an object while it is being - defined, e.g. to add version or dependency information. Directives are - one of the key pieces of Ramble's object "language", which is - embedded in python. - - Here's an example directive: - - .. code-block:: python - - @directive(dicts='workloads') - workload('workload_name', ...): - ... - - This directive allows you write: - - .. code-block:: python + def __prepare__(metacls, *args: Any, **kwds: Any) -> Any: + DirectiveMeta._directives_to_be_executed.clear() + DirectiveMeta._when_constraints_from_context.clear() + DirectiveMeta._default_args.clear() + return super().__prepare__(*args, **kwds) + + def __new__( + cls: Type["DirectiveMeta"], name: str, bases: tuple, attr_dict: dict + ) -> "DirectiveMeta": + # Initialize the attribute containing the list of directives + # to be executed following MRO order and class definition order. + merged: List[Tuple[str, Any]] = [] + sources = [getattr(b, "_directives_to_be_executed", None) or [] for b in reversed(bases)] + for source in sources: + merged.extend(source) + + defining_id = object() + for _, directive in DirectiveMeta._directives_to_be_executed: + try: + directive._defining_class = defining_id + except (AttributeError, TypeError): + pass + + merged.extend(DirectiveMeta._directives_to_be_executed) + DirectiveMeta._directives_to_be_executed.clear() + + # Deduplicate directives by callable identity to prevent double execution + seen_fns = set() + deduped: List[Tuple[str, Any]] = [] + for directive_name, fn in merged: + if fn not in seen_fns: + seen_fns.add(fn) + deduped.append((directive_name, fn)) + merged = deduped + + attr_dict["_directives_to_be_executed"] = merged + + # Determine language types to scope descriptors + lang_types = set(attr_dict.get("_language_types", [])) | set( + attr_dict.get("_language_classes", []) + ) + for base in bases: + lang_types |= set(getattr(base, "_language_types", [])) | set( + getattr(base, "_language_classes", []) + ) - class Foo(ApplicationBase): - workload(...) + if lang_types: + relevant_dicts = set() + for d in DirectiveMeta._directive_dict_names: + scoped_types = [ + t for t, dicts in DirectiveMeta._type_scoped_dicts.items() if d in dicts + ] + if not scoped_types or any(t in lang_types for t in scoped_types): + relevant_dicts.add(d) + else: + relevant_dicts = set(DirectiveMeta._directive_dict_names) + + # Collect class-level attribute initial values + class_directive_values = {} + for base in bases: + if hasattr(base, "_class_directive_values"): + class_directive_values.update(base._class_directive_values) + + # Add descriptors for known directive dictionaries + for dict_name in relevant_dicts: + if dict_name in attr_dict and attr_dict[ + dict_name + ] is not DirectiveMeta._get_descriptor(dict_name): + val = attr_dict.pop(dict_name) + default_init = DirectiveMeta._directive_init_values.get(dict_name, {}) + if default_init is None or isinstance(val, type(default_init)): + class_directive_values[dict_name] = val + attr_dict.setdefault(f"_{dict_name}", _UNSET) + attr_dict[dict_name] = DirectiveMeta._get_descriptor(dict_name) + + attr_dict["_class_directive_values"] = class_directive_values + + attr_dict["_directive_functions"] = dict(DirectiveMeta._directive_functions) + attr_dict["_directive_classes"] = dict(DirectiveMeta._directive_classes) + attr_dict["_directive_types"] = dict(DirectiveMeta._directive_types) + attr_dict["_directive_dict_names"] = relevant_dicts - The ``@directive`` decorator handles a couple things for you: + return super().__new__(cls, name, bases, attr_dict) - 1. Adds the class scope (app) as an initial parameter when - called, like a class method would. This allows you to modify - a package from within a directive, while the package is still - being defined. + def __init__(cls: "DirectiveMeta", name: str, bases: tuple, attr_dict: dict) -> None: + super().__init__(name, bases, attr_dict) + directives.define_directive_methods_on_class(cls) + + # Execute eager directives (directives without target dictionaries) + DirectiveMeta._executing_directives_depth += 1 + try: + for directive_name, directive in getattr(cls, "_directives_to_be_executed", []): + if not DirectiveMeta._directive_to_dicts.get(directive_name): + directive(cls) + finally: + DirectiveMeta._executing_directives_depth -= 1 + + def __setattr__(cls: "DirectiveMeta", name: str, value: Any) -> None: + if name in DirectiveMeta._directive_dict_names: + super().__setattr__(f"_{name}", value) + else: + super().__setattr__(name, value) - 2. It automatically adds a dictionary called "workloads" to the - package so that you can refer to app.workloads. + @classmethod + def register_directive(cls, name: str, dicts: Tuple[str, ...]) -> None: + """Called by directive decorator to register relationships.""" + DirectiveMeta._execution_plan_cache.clear() + DirectiveMeta._directive_to_dicts[name] = dicts + for d in dicts: + if name not in DirectiveMeta._dict_to_directives[d]: + DirectiveMeta._dict_to_directives[d].append(name) + + @staticmethod + def _get_descriptor(name: str) -> DirectiveDictDescriptor: + """Returns a singleton descriptor for the given dictionary name.""" + if name not in DirectiveMeta._descriptor_cache: + DirectiveMeta._descriptor_cache[name] = DirectiveDictDescriptor(name) + return DirectiveMeta._descriptor_cache[name] + + @staticmethod + def get_cached_execution_plan(target_dict: str) -> Tuple[List[str], Set[str]]: + """Returns cached execution plan with directives as a set for O(1) membership check.""" + if target_dict not in DirectiveMeta._execution_plan_cache: + dicts_to_init, directives_to_run = DirectiveMeta._get_execution_plan(target_dict) + DirectiveMeta._execution_plan_cache[target_dict] = ( + dicts_to_init, + set(directives_to_run), + ) + return DirectiveMeta._execution_plan_cache[target_dict] + + @property + def preferred_version(cls: "DirectiveMeta") -> Optional[Any]: + for ver in getattr(cls, "known_versions", {}).values(): + if getattr(ver, "preferred", False): + return ver + return None + + @preferred_version.setter + def preferred_version(cls: "DirectiveMeta", value: Optional[Any]) -> None: + if not hasattr(cls, "known_versions"): + return + for ver in cls.known_versions.values(): + if hasattr(ver, "preferred"): + ver.preferred = False + if value is not None: + try: + value.preferred = True + except (AttributeError, TypeError): + pass + ver_key = ( + getattr(value, "version_number", None) + or getattr(value, "version", None) + or str(value) + ) + cls.known_versions[ver_key] = value - The ``(dicts='workloads')`` part ensures that ALL applications in - Ramble will have a ``workloads`` attribute after they're constructed, - and that if no directive actually modified it, it will just be an empty - dict. + @staticmethod + def _get_execution_plan(target_dict: str) -> Tuple[List[str], List[str]]: + """Calculates the closure of dicts and directives needed to populate target_dict.""" + dicts_involved = {target_dict} + directives_involved: List[str] = [] + stack = [target_dict] - The ``(init_value={})`` part allows objects in Ramble to define what the - type of the attribute defined by the `dicts` argument will be. This - allows ``(dicts="variables", init_value=[])`` which makes the attribute - a list instead of a dict, which is the default. + while stack: + current_dict = stack.pop() - This is just a modular way to add storage attributes to the Application - class, and it's how Ramble gets information from the applications to - the core. + for directive_name in DirectiveMeta._dict_to_directives.get(current_dict, ()): + if directive_name in directives_involved: + continue - """ - if isinstance(dicts, str): - dicts = (dicts,) + directives_involved.append(directive_name) - if not isinstance(dicts, Sequence): - message = "dicts arg must be list, tuple, or string. Found {0}" - raise TypeError(message.format(type(dicts))) + for other_dict in DirectiveMeta._directive_to_dicts.get(directive_name, ()): + if other_dict not in dicts_involved: + dicts_involved.add(other_dict) + stack.append(other_dict) - if init_value is None: - init_value = {} + return sorted(dicts_involved), directives_involved - # Add the dictionary names if not already there - dicts_set = set(dicts) - cls._directive_names |= dicts_set - for attr_name in dicts_set: - cls._directive_init_values[attr_name] = init_value + @classmethod + def directive( + cls: Type["DirectiveMeta"], + dicts: Union[Sequence[str], str, None] = None, + init_value: Any = _UNSET, + language_type: str = "shared", + ) -> Callable[..., Any]: + """Decorator for Ramble directives.""" + if dicts is None or dicts == (): + dicts_tuple: Tuple[str, ...] = () + elif isinstance(dicts, str): + dicts_tuple = (dicts,) + elif isinstance(dicts, Sequence): + dicts_tuple = tuple(dicts) + else: + message = f"dicts arg must be list, tuple, or string. Found {type(dicts)}" + raise TypeError(message) + + # Add the dictionary names and auto-register type scoping + for attr_name in dicts_tuple: + DirectiveMeta._directive_dict_names.add(attr_name) + if init_value is not _UNSET: + DirectiveMeta._directive_init_values[attr_name] = init_value + elif attr_name not in DirectiveMeta._directive_init_values: + DirectiveMeta._directive_init_values[attr_name] = {} + + if language_type == "shared": + if attr_name not in DirectiveMeta._cross_boundary_app_dicts: + DirectiveMeta._shared_dict_names.add(attr_name) + for type_set in DirectiveMeta._type_scoped_dicts.values(): + type_set.discard(attr_name) + else: + DirectiveMeta._type_scoped_dicts["application"].add(attr_name) + else: + if attr_name not in DirectiveMeta._shared_dict_names: + DirectiveMeta._type_scoped_dicts[language_type].add(attr_name) + + def _decorator(decorated_function: Callable[..., Any]) -> Callable[..., Any]: + func_name = decorated_function.__name__ + DirectiveMeta.register_directive(func_name, dicts_tuple) + DirectiveMeta._directive_classes[func_name] = cls + DirectiveMeta._directive_types[func_name] = language_type + DirectiveMeta._directive_functions[func_name] = decorated_function - # This decorator just returns the directive functions - def _decorator(decorated_function): @functools.wraps(decorated_function) - def _wrapper(*args, **_kwargs): + def _wrapper(*args: Any, **_kwargs: Any) -> Any: # First merge default args with kwargs - kwargs = {} - for default_args in DirectiveMeta._default_args: - kwargs.update(default_args) - kwargs.update(_kwargs) + if DirectiveMeta._default_args: + kwargs = {} + for default_args in DirectiveMeta._default_args: + kwargs.update(default_args) + kwargs.update(_kwargs) + else: + kwargs = _kwargs # Inject when arguments from the context if DirectiveMeta._when_constraints_from_context: - # Check that directives not yet supporting the when= argument - # are not used inside the context manager sig = inspect.signature(decorated_function) if "when" not in sig.parameters: msg = ( - 'directive "{0}" cannot be used within a "when"' - ' context since it does not support a "when=" ' - "argument" + f'directive "{decorated_function.__name__}" cannot be used ' + 'within a "when" context since it does not support a "when=" argument' ) - msg = msg.format(decorated_function.__name__) raise DirectiveError(msg) - when_constraints = DirectiveMeta._when_constraints_from_context.copy() + when_constraints = list(DirectiveMeta._when_constraints_from_context) if kwargs.get("when"): when_arg = kwargs["when"] - - # Validate kwarg `when` conditions are correctly formatted + directive_id = str(args[0]) if args else "" when_list = ramble.language.language_helpers.build_when_list( - when_arg, "DirectiveMeta", args[0], decorated_function.__name__ + when_arg, + "DirectiveMeta", + directive_id, + decorated_function.__name__, ) - when_constraints.extend(when_list) - kwargs["when"] = when_constraints.copy() + kwargs["when"] = when_constraints if "when" in kwargs: impossible, message = ramble.language.language_helpers.is_when_impossible( @@ -304,8 +451,8 @@ def _wrapper(*args, **_kwargs): if impossible: def _warn_impossible(obj): - obj_type = obj.origin_type if hasattr(obj, "origin_type") else "" - obj_name = obj.name if hasattr(obj, "name") else "" + obj_type = getattr(obj, "origin_type", "") + obj_name = getattr(obj, "name", "") _impossible_when_warning( decorated_function.__name__, obj_type, @@ -315,49 +462,40 @@ def _warn_impossible(obj): kwargs, ) - DirectiveMeta._directives_to_be_executed.append(_warn_impossible) + DirectiveMeta._directives_to_be_executed.append( + (func_name, _warn_impossible) + ) return _warn_impossible - # If any of the arguments are executors returned by a - # directive passed as an argument, don't execute them - # lazily. Instead, let the called directive handle them. - # This allows nested directive calls in applications. The - # caller can return the directive if it should be queued. + # Handle nested directives passed as arguments def remove_directives(arg): - directives = DirectiveMeta._directives_to_be_executed - if isinstance(arg, (list, tuple)): - # Descend into args that are lists or tuples + if isinstance(arg, (list, tuple, set)): for a in arg: remove_directives(a) - else: - # Remove directives args from the exec queue - remove = next((d for d in directives if d is arg), None) - if remove is not None: - directives.remove(remove) + elif isinstance(arg, dict): + for a in arg.values(): + remove_directives(a) + elif callable(arg): + DirectiveMeta._directives_to_be_executed = [ + (n, fn) + for n, fn in DirectiveMeta._directives_to_be_executed + if fn is not arg + ] - # Nasty, but it's the best way I can think of to avoid - # side effects if directive results are passed as args remove_directives(args) remove_directives(list(kwargs.values())) - # A directive returns either something that is callable on a - # package or a sequence of them result = decorated_function(*args, **kwargs) - # ...so if it is not a sequence make it so - values = result - if not isinstance(values, Sequence): - values = (values,) - - DirectiveMeta._directives_to_be_executed.extend(values) + if result is not None and DirectiveMeta._executing_directives_depth == 0: + if isinstance(result, Sequence) and not isinstance(result, (str, bytes)): + for item in result: + DirectiveMeta._directives_to_be_executed.append((func_name, item)) + else: + DirectiveMeta._directives_to_be_executed.append((func_name, result)) - # wrapped function returns same result as original so - # that we can nest directives return result - cls._directive_classes[decorated_function.__name__] = cls - cls._directive_functions[decorated_function.__name__] = decorated_function - return _wrapper return _decorator diff --git a/lib/ramble/ramble/language/modifier_language.py b/lib/ramble/ramble/language/modifier_language.py index 0944f015f5..eda7bb3409 100644 --- a/lib/ramble/ramble/language/modifier_language.py +++ b/lib/ramble/ramble/language/modifier_language.py @@ -6,6 +6,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools from typing import Optional import ramble.definitions.requirements @@ -14,13 +15,8 @@ from ramble.definitions.variables import EnvironmentVariableModifications, VariableModification from ramble.error import DirectiveError - -class ModifierMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - - -modifier_directive = ModifierMeta.directive +ModifierMeta = ramble.language.shared_language.SharedMeta +modifier_directive = functools.partial(ModifierMeta.directive, language_type="modifier") @modifier_directive("modes") @@ -47,7 +43,7 @@ def _execute_mode(mod): return _execute_mode -@modifier_directive(dicts=()) +@modifier_directive(dicts="default_usage_mode", init_value=None) def default_mode(name, **kwargs): """Define a default mode for this modifier. @@ -69,7 +65,7 @@ def _execute_default_mode(mod): f"default_mode directive given an invalid mode for modifier " f"{mod.name}. The disabled mode cannot be set as the default mode" ) - mod._default_usage_mode = name + mod.default_usage_mode = name return _execute_default_mode @@ -265,15 +261,16 @@ def _env_var_modification(mod): ) else: mod.env_var_modifications[when_set][name].add_modification( - modification=modification, - method=method, - **kwargs, + modification=modification, method=method, when=when_list, **kwargs ) return _env_var_modification -@modifier_directive(dicts=()) +@modifier_directive( + dicts=("object_variables", "object_environment_variables", "validators"), + init_value={}, +) def modifier_variable( name: str, default, diff --git a/lib/ramble/ramble/language/package_manager_language.py b/lib/ramble/ramble/language/package_manager_language.py index c2118f7467..fb958abeac 100644 --- a/lib/ramble/ramble/language/package_manager_language.py +++ b/lib/ramble/ramble/language/package_manager_language.py @@ -6,20 +6,21 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools from typing import Optional import ramble.language.shared_language +PackageManagerMeta = ramble.language.shared_language.SharedMeta +package_manager_directive = functools.partial( + PackageManagerMeta.directive, language_type="package_manager" +) -class PackageManagerMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - -package_manager_directive = PackageManagerMeta.directive - - -@package_manager_directive(dicts=()) +@package_manager_directive( + dicts=("object_variables", "object_environment_variables", "validators"), + init_value={}, +) def package_manager_variable( name: str, default, @@ -59,7 +60,7 @@ def _define_package_manager_variable(pm): return _define_package_manager_variable -@package_manager_directive(dicts=()) +@package_manager_directive(dicts="class_families", init_value={}) def package_manager_family(*names: str, **kwargs): """Add a new family to this package manager diff --git a/lib/ramble/ramble/language/platform_language.py b/lib/ramble/ramble/language/platform_language.py index ee7bdb91ec..fd742d9e0c 100644 --- a/lib/ramble/ramble/language/platform_language.py +++ b/lib/ramble/ramble/language/platform_language.py @@ -6,6 +6,8 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools + import ramble.language.shared_language """This package contains directives that can be used within a platform. @@ -21,16 +23,11 @@ class MyPlatform(Platform): In the above example, 'platform_family' is a ramble directive """ - -class PlatformMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - - -platform_directive = PlatformMeta.directive +PlatformMeta = ramble.language.shared_language.SharedMeta +platform_directive = functools.partial(PlatformMeta.directive, language_type="platform") -@platform_directive(dicts=()) +@platform_directive(dicts="class_families", init_value={}) def platform_family(*names: str, **kwargs): """Add a new family to this platform diff --git a/lib/ramble/ramble/language/shared_language.py b/lib/ramble/ramble/language/shared_language.py index 673c02339e..25bd19f322 100644 --- a/lib/ramble/ramble/language/shared_language.py +++ b/lib/ramble/ramble/language/shared_language.py @@ -8,6 +8,7 @@ import collections import contextlib +import functools from typing import Any, Callable, List, Optional, Union import ramble.language.language_base @@ -37,13 +38,8 @@ class Gromacs(ExecutableApplication): """ -class SharedMeta(ramble.language.language_base.DirectiveMeta): - _directive_names = set() - _directives_to_be_executed = [] - _directive_init_values = {"custom_edit_functions": {}} - - -shared_directive = SharedMeta.directive +SharedMeta = ramble.language.language_base.DirectiveMeta +shared_directive = functools.partial(SharedMeta.directive, language_type="shared") def _add_specs( @@ -88,7 +84,7 @@ def _add_list_attributes(obj, attr_name, values): setattr(obj, attr_name, sorted(set(base_list + list(values)))) -@shared_directive("executables") +@shared_directive(dicts=("executables", "custom_edit_functions"), init_value={}) def edit_file( name, file_path, @@ -819,7 +815,7 @@ def _execute_register_phase(obj): return _execute_register_phase -@shared_directive(dicts=()) +@shared_directive(dicts="maintainers", init_value=[]) def maintainers(*names: str, **kwargs): """Add a new maintainer directive, to specify maintainers in a declarative way. @@ -834,7 +830,7 @@ def _execute_maintainers(obj): return _execute_maintainers -@shared_directive(dicts=()) +@shared_directive(dicts="tags", init_value=[]) def tags(*values: str, **kwargs): """Add a new tag directive, to specify tags in a declarative way. @@ -864,7 +860,7 @@ def _define_class_family(obj): return _define_class_family -@shared_directive(dicts=()) +@shared_directive(dicts="shell_support_pattern", init_value=None) def target_shells(shell_support_pattern=None, **kwargs): """Directive to specify supported shells. @@ -1108,7 +1104,7 @@ def _execute_conflicts(obj): return _execute_conflicts -@shared_directive("object_variables") +@shared_directive(dicts=("object_variables", "object_environment_variables", "validators")) def variable( name: str, default, @@ -1191,7 +1187,14 @@ def _define_variable(obj): return _define_variable -@shared_directive(dicts=("workload_group_env_vars", "object_environment_variables")) +@shared_directive( + dicts=( + "object_environment_variables", + "workload_group_env_vars", + "workload_groups", + "workloads", + ) +) def environment_variable( name, value, @@ -1341,7 +1344,7 @@ def _define_variant(obj): return _define_variant -@shared_directive("scripts_to_source") +@shared_directive(dicts="scripts_to_source", init_value=[]) def source_script( script_path: str, when=None, @@ -1390,22 +1393,30 @@ def _define_version(obj): # Ensure only one version is marked as preferred if new_version.preferred: - if not hasattr(obj, "preferred_version"): - obj.preferred_version = new_version - elif obj.preferred_version.version == new_version.version: + curr_preferred = getattr(obj, "preferred_version", None) + this_class = getattr(_define_version, "_defining_class", None) + curr_class = ( + getattr(curr_preferred, "_defining_class", None) if curr_preferred else None + ) + + if curr_preferred is None or (this_class is not None and curr_class != this_class): + for ver in obj.known_versions.values(): + ver.preferred = False + new_version._defining_class = this_class + elif curr_preferred.version == new_version.version: # Ignore identical preferred versions, which happens when app is subclassed pass else: raise ramble.language.language_base.DirectiveError( f"Object {obj.name} already has a preferred version " - f"({obj.preferred_version.version}). Only one version can be marked preferred." + f"({curr_preferred.version}). Only one version can be marked preferred." ) obj.known_versions[number] = new_version return _define_version -@shared_directive(dicts=()) +@shared_directive(dicts="enable_strict_versions", init_value=True) def strict_versions(strict: bool = True, **kwargs): """Directive to specify if the object has strict versioning. If true, only known versions can be used in experiments. @@ -1420,7 +1431,7 @@ def _execute_strict_versions(obj): return _execute_strict_versions -@shared_directive("required_vars") +@shared_directive(dicts="required_vars") def required_variable( var: str, results_level="variable", diff --git a/lib/ramble/ramble/language/system_language.py b/lib/ramble/ramble/language/system_language.py index a37f6ffa34..2f4020d9b3 100644 --- a/lib/ramble/ramble/language/system_language.py +++ b/lib/ramble/ramble/language/system_language.py @@ -6,6 +6,8 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools + import ramble.language.shared_language """This package contains directives that can be used within a system. @@ -21,16 +23,11 @@ class MySystem(System): In the above example, 'default_platform' is a ramble directive """ - -class SystemMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - - -system_directive = SystemMeta.directive +SystemMeta = ramble.language.shared_language.SharedMeta +system_directive = functools.partial(SystemMeta.directive, language_type="system") -@system_directive("default_workflow_managers") +@system_directive(dicts="system_default_workflow_manager", init_value=None) def default_workflow_manager(name, **kwargs): """Sets the default workflow manager for this system @@ -44,7 +41,7 @@ def _execute_default_workflow_manager(obj): return _execute_default_workflow_manager -@system_directive(dicts=()) +@system_directive(dicts="system_default_package_manager", init_value=None) def default_package_manager(name, **kwargs): """Sets the default package manager for this system @@ -58,7 +55,7 @@ def _execute_default_package_manager(obj): return _execute_default_package_manager -@system_directive(dicts=()) +@system_directive(dicts="system_default_platform", init_value=None) def default_platform(name, **kwargs): """Sets the default platform for this system @@ -72,7 +69,7 @@ def _execute_default_platform(obj): return _execute_default_platform -@system_directive(dicts=()) +@system_directive(dicts="system_available_platforms", init_value=[]) def available_platforms(platforms, **kwargs): """Sets the available platforms for this system @@ -93,7 +90,7 @@ def _execute_available_platforms(obj): return _execute_available_platforms -@system_directive(dicts="platform_variable_maps") +@system_directive(dicts="platform_variable_maps", init_value={}) def platform_variable_map(variable_name, var_map, **kwargs): """Defines a mapping of platform to variable values @@ -110,7 +107,7 @@ def _execute_platform_variable_map(obj): return _execute_platform_variable_map -@system_directive(dicts="variable_defaults") +@system_directive(dicts="variable_defaults", init_value={}) def variable_defaults(variable_definitions, when=None, **kwargs): """Defines default values for variables @@ -131,7 +128,7 @@ def _execute_variable_defaults(obj): return _execute_variable_defaults -@system_directive(dicts=()) +@system_directive(dicts="class_families", init_value={}) def system_family(*names: str, **kwargs): """Add a new family to this system diff --git a/lib/ramble/ramble/language/utility_language.py b/lib/ramble/ramble/language/utility_language.py index 0c89d80083..c4f2afb029 100644 --- a/lib/ramble/ramble/language/utility_language.py +++ b/lib/ramble/ramble/language/utility_language.py @@ -6,18 +6,14 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools from typing import Optional import ramble.language.language_helpers import ramble.language.shared_language - -class UtilityMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - - -utility_directive = UtilityMeta.directive +UtilityMeta = ramble.language.shared_language.SharedMeta +utility_directive = functools.partial(UtilityMeta.directive, language_type="utility") @utility_directive("env_sources") diff --git a/lib/ramble/ramble/language/workflow_manager_language.py b/lib/ramble/ramble/language/workflow_manager_language.py index 0988a7f0d9..5c198ec64b 100644 --- a/lib/ramble/ramble/language/workflow_manager_language.py +++ b/lib/ramble/ramble/language/workflow_manager_language.py @@ -6,20 +6,21 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import functools from typing import Optional import ramble.language.shared_language +WorkflowManagerMeta = ramble.language.shared_language.SharedMeta +workflow_manager_directive = functools.partial( + WorkflowManagerMeta.directive, language_type="workflow_manager" +) -class WorkflowManagerMeta(ramble.language.shared_language.SharedMeta): - _directive_names = set() - _directives_to_be_executed = [] - -workflow_manager_directive = WorkflowManagerMeta.directive - - -@workflow_manager_directive(dicts=()) +@workflow_manager_directive( + dicts=("object_variables", "object_environment_variables", "validators"), + init_value={}, +) def workflow_manager_variable( name: str, default, @@ -59,7 +60,7 @@ def _define_wm_variable(wm): return _define_wm_variable -@workflow_manager_directive(dicts=()) +@workflow_manager_directive(dicts="class_families", init_value={}) def workflow_manager_family(*names: str, **kwargs): """Add a new family to this workflow manager diff --git a/lib/ramble/ramble/test/language/test_lazy_directives.py b/lib/ramble/ramble/test/language/test_lazy_directives.py new file mode 100644 index 0000000000..f239ff1e61 --- /dev/null +++ b/lib/ramble/ramble/test/language/test_lazy_directives.py @@ -0,0 +1,714 @@ +# Copyright 2022-2026 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import pytest + +import ramble.language.application_language +import ramble.language.shared_language +from ramble.appkit import ExecutableApplication +from ramble.language.language_base import _UNSET, DirectiveError, DirectiveMeta + + +@pytest.fixture(autouse=True) +def isolate_directive_registry(): + """Snapshot and restore DirectiveMeta global registries around each test.""" + import ramble.language.application_language # noqa: F401 + import ramble.language.modifier_language # noqa: F401 + import ramble.language.package_manager_language # noqa: F401 + import ramble.language.platform_language # noqa: F401 + import ramble.language.shared_language # noqa: F401 + import ramble.language.system_language # noqa: F401 + import ramble.language.utility_language # noqa: F401 + import ramble.language.workflow_manager_language # noqa: F401 + + saved_dict_names = set(DirectiveMeta._directive_dict_names) + saved_init_values = dict(DirectiveMeta._directive_init_values) + saved_to_dicts = dict(DirectiveMeta._directive_to_dicts) + saved_dict_to_dirs = {k: list(v) for k, v in DirectiveMeta._dict_to_directives.items()} + saved_functions = dict(DirectiveMeta._directive_functions) + saved_classes = dict(DirectiveMeta._directive_classes) + saved_types = dict(DirectiveMeta._directive_types) + saved_type_scoped = {k: set(v) for k, v in DirectiveMeta._type_scoped_dicts.items()} + saved_shared = set(DirectiveMeta._shared_dict_names) + saved_descriptor_cache = dict(DirectiveMeta._descriptor_cache) + + yield + + DirectiveMeta._directive_dict_names.clear() + DirectiveMeta._directive_dict_names.update(saved_dict_names) + DirectiveMeta._directive_init_values.clear() + DirectiveMeta._directive_init_values.update(saved_init_values) + DirectiveMeta._directive_to_dicts.clear() + DirectiveMeta._directive_to_dicts.update(saved_to_dicts) + DirectiveMeta._dict_to_directives.clear() + for k, v in saved_dict_to_dirs.items(): + DirectiveMeta._dict_to_directives[k] = v + DirectiveMeta._directive_functions.clear() + DirectiveMeta._directive_functions.update(saved_functions) + DirectiveMeta._directive_classes.clear() + DirectiveMeta._directive_classes.update(saved_classes) + DirectiveMeta._directive_types.clear() + DirectiveMeta._directive_types.update(saved_types) + DirectiveMeta._type_scoped_dicts.clear() + for k, v in saved_type_scoped.items(): + DirectiveMeta._type_scoped_dicts[k] = v + DirectiveMeta._shared_dict_names.clear() + DirectiveMeta._shared_dict_names.update(saved_shared) + DirectiveMeta._descriptor_cache.clear() + DirectiveMeta._descriptor_cache.update(saved_descriptor_cache) + DirectiveMeta._execution_plan_cache.clear() + + +def test_lazy_directive_evaluation(): + """Verify that directives are not evaluated until attributes are accessed.""" + + class LazyTestApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "lazy_test_app" + __module__ = "ramble.app" + + ramble.language.shared_language.tags("tag1", "tag2") + ramble.language.application_language.workload("test_wl", executables=["exe1"]) + + assert LazyTestApp._workloads is _UNSET + assert LazyTestApp._tags is _UNSET + + wl = LazyTestApp.workloads + assert LazyTestApp._workloads is not _UNSET + assert frozenset() in wl + assert "test_wl" in wl[frozenset()] + + tags = LazyTestApp.tags + assert LazyTestApp._tags is not _UNSET + assert "tag1" in tags + assert "tag2" in tags + + +def test_lazy_directive_inheritance_and_mro(): + """Verify inheritance and MRO order when directives are executed lazily.""" + + class ParentApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "parent_app" + __module__ = "ramble.app" + + ramble.language.application_language.workload("parent_wl", executables=["p_exe"]) + ramble.language.shared_language.maintainers("parent_user") + + class ChildApp(ParentApp): + name = "child_app" + __module__ = "ramble.app" + + ramble.language.application_language.workload("child_wl", executables=["c_exe"]) + ramble.language.shared_language.maintainers("child_user") + + assert ChildApp._workloads is _UNSET + assert ChildApp._maintainers is _UNSET + + child_wls = ChildApp.workloads + assert frozenset() in child_wls + assert "parent_wl" in child_wls[frozenset()] + assert "child_wl" in child_wls[frozenset()] + + child_maintainers = ChildApp.maintainers + assert "parent_user" in child_maintainers + assert "child_user" in child_maintainers + + +def test_instance_attribute_isolation(): + """Verify copy-on-first-access isolates instance attributes from class descriptors.""" + + class IsolatedApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "isolated_app" + __module__ = "ramble.app" + + ramble.language.application_language.workload("base_wl", executables=["base_exe"]) + + inst = IsolatedApp() + inst2 = IsolatedApp() + + assert "workloads" not in inst.__dict__ + assert "workloads" not in inst2.__dict__ + assert IsolatedApp._workloads is _UNSET + + inst_wl = inst.workloads + assert "workloads" in inst.__dict__ + assert "workloads" not in inst2.__dict__ + assert "base_wl" in inst_wl[frozenset()] + + inst.workloads[frozenset()]["inst_wl"] = {"executables": ["inst_exe"]} + + assert "inst_wl" not in IsolatedApp.workloads[frozenset()] + assert "base_wl" in IsolatedApp.workloads[frozenset()] + assert "inst_wl" in inst.workloads[frozenset()] + assert "inst_wl" not in inst2.workloads[frozenset()] + + +def test_graph_closure_multi_dict_execution(): + """Verify that accessing one dictionary triggers all co-dependent directives.""" + + class MultiDictApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "multi_dict_app" + __module__ = "ramble.app" + + ramble.language.shared_language.edit_file( + name="patch_cfg", + file_path="config.txt", + match="FOO", + replace="BAR", + ) + + assert MultiDictApp._executables is _UNSET + assert MultiDictApp._custom_edit_functions is _UNSET + + exes = MultiDictApp.executables + assert frozenset() in exes + assert "patch_cfg" in exes[frozenset()] + assert MultiDictApp._custom_edit_functions is not _UNSET + + +def test_modifier_lazy_directives(): + """Verify lazy evaluation of modifier-specific directives.""" + import ramble.language.modifier_language + + class LazyMod(metaclass=ramble.language.modifier_language.ModifierMeta): + name = "lazy_mod" + __module__ = "ramble.mod" + + ramble.language.modifier_language.mode("opt", description="Optimized mode") + ramble.language.modifier_language.default_mode("opt") + ramble.language.modifier_language.modifier_variable( + "threads", default="4", description="Thread count" + ) + + assert LazyMod._modes is _UNSET + assert LazyMod._default_usage_mode is _UNSET + assert LazyMod._object_variables is _UNSET + + modes = LazyMod.modes + assert "opt" in modes + assert LazyMod.default_usage_mode == "opt" + + vars_dict = LazyMod.object_variables + assert frozenset() in vars_dict + assert "threads" in [v.name for v in vars_dict[frozenset()]] + + +def test_system_lazy_directives(): + """Verify lazy evaluation of system-specific directives.""" + import ramble.language.system_language + + class LazySys(metaclass=ramble.language.system_language.SystemMeta): + name = "lazy_sys" + __module__ = "ramble.sys" + + ramble.language.system_language.default_platform("x86_64") + ramble.language.system_language.available_platforms(["x86_64", "arm64"]) + ramble.language.system_language.variable_defaults({"n_ranks": "16"}) + + assert LazySys._system_default_platform is _UNSET + assert LazySys._system_available_platforms is _UNSET + assert LazySys._variable_defaults is _UNSET + + assert LazySys.system_default_platform == "x86_64" + assert "arm64" in LazySys.system_available_platforms + assert frozenset() in LazySys.variable_defaults + assert LazySys.variable_defaults[frozenset()]["n_ranks"] == "16" + + +def test_dynamic_instance_directive_execution(): + """Verify dynamic invocation of directive methods on instances.""" + + class DynamicApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "dynamic_app" + __module__ = "ramble.app" + _language_types = ["application", "shared"] + _language_classes = _language_types + + ramble.language.application_language.workload("static_wl", executables=["static_exe"]) + + inst = DynamicApp() + + inst.workload("dynamic_wl", executables=["dynamic_exe"]) + + assert "dynamic_wl" in inst.workloads[frozenset()] + assert "dynamic_wl" not in DynamicApp.workloads[frozenset()] + + +def test_application_clone_preserves_evaluated_directives(mutable_mock_apps_repo): + """Verify application clone preserves evaluated and dynamically added directives.""" + app = mutable_mock_apps_repo.get("basic") + app.set_variables_and_variants({"workload_name": "test_wl"}, {}, None, None) + app.workload("dyn_wl", executables=["dyn_exe"]) + + assert "dyn_wl" in app.workloads[frozenset()] + + clone = app.clone() + + assert "dyn_wl" in clone.workloads[frozenset()] + assert "archive_patterns" not in clone.__dict__ + + +def test_generic_object_copy_preserves_evaluated_directives(mutable_mock_mods_repo): + """Verify generic Ramble object copy (e.g. modifier) preserves evaluated directives.""" + mod = mutable_mock_mods_repo.get("spack-mod") + _ = mod.modes + assert "modes" in mod.__dict__ + + mod_copy = mod.copy() + assert "modes" in mod_copy.__dict__ + assert mod_copy.modes == mod.modes + assert "env_var_modifications" not in mod_copy.__dict__ + + +def test_subclass_directive_evaluation_when_parent_already_evaluated(): + """Verify that evaluating parent directives first does not prevent subclass evaluation.""" + + class ParentApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "parent_eval_app" + __module__ = "ramble.app" + _language_types = ["application", "shared"] + ramble.language.application_language.workload("parent_wl", executables=["p_exe"]) + + class ChildApp(ParentApp): + name = "child_eval_app" + __module__ = "ramble.app" + _language_types = ["application", "shared"] + ramble.language.application_language.workload("child_wl", executables=["c_exe"]) + + parent_wls = ParentApp.workloads + assert "parent_wl" in parent_wls[frozenset()] + assert "child_wl" not in parent_wls[frozenset()] + + child_wls = ChildApp.workloads + assert "parent_wl" in child_wls[frozenset()] + assert "child_wl" in child_wls[frozenset()] + + +def test_subclass_preferred_version_override(): + """Verify that a subclass can override the parent's preferred version.""" + + class ParentVerApp(ExecutableApplication): + name = "parent_ver_app" + __module__ = "ramble.app" + ramble.language.shared_language.version("1.0", preferred=True) + + class ChildVerApp(ParentVerApp): + name = "child_ver_app" + __module__ = "ramble.app" + ramble.language.shared_language.version("2.0", preferred=True) + + assert str(ParentVerApp.preferred_version.version) == "1.0" + assert str(ChildVerApp.preferred_version.version) == "2.0" + + with pytest.raises(DirectiveError, match="already has a preferred version"): + + class ConflictVerApp(ExecutableApplication): + name = "conflict_ver_app" + __module__ = "ramble.app" + ramble.language.shared_language.version("1.0", preferred=True) + ramble.language.shared_language.version("2.0", preferred=True) + + _ = ConflictVerApp.preferred_version + + +def test_class_level_attribute_preservation(): + """Verify that class-level attributes matching directive names are preserved.""" + + class ClassAttrApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "class_attr_app" + __module__ = "ramble.app" + maintainers = ["alice", "bob"] + tags = ["tag_custom"] + + assert ClassAttrApp.maintainers == ["alice", "bob"] + assert ClassAttrApp.tags == ["tag_custom"] + + +def test_instance_preferred_version_preservation_on_clone(mutable_mock_apps_repo): + """Verify that setting preferred_version on an instance is preserved across clone.""" + import spack.version + + app = mutable_mock_apps_repo.get("basic") + app.set_variables_and_variants({"workload_name": "test_wl"}, {}, None, None) + + class CustomVer: + version = spack.version.Version("99.99") + + app.preferred_version = CustomVer() + assert str(app.preferred_version.version) == "99.99" + + clone = app.clone() + assert clone.preferred_version is not None + assert str(clone.preferred_version.version) == "99.99" + + +def test_family_directive_single_execution(): + """Verify that family directives do not execute multiple times.""" + import ramble.language.workflow_manager_language + + class TestWM(metaclass=ramble.language.workflow_manager_language.WorkflowManagerMeta): + name = "test_wm" + __module__ = "ramble.wm" + ramble.language.workflow_manager_language.workflow_manager_family("fam_test") + + exec_count = 0 + wrapped_list = [] + for d_name, d_fn in TestWM._directives_to_be_executed: + + def make_wrapper(target_fn): + def _wrapped(cls): + nonlocal exec_count + exec_count += 1 + return target_fn(cls) + + return _wrapped + + wrapped_list.append((d_name, make_wrapper(d_fn))) + + TestWM._directives_to_be_executed = wrapped_list + assert "fam_test" in TestWM.class_families + assert exec_count == 1 + + +def test_language_types_descriptor_scoping(mutable_mock_apps_repo, mutable_mock_mods_repo): + """Verify descriptors are scoped to declared language types.""" + app = mutable_mock_apps_repo.get("basic") + assert hasattr(app, "workloads") + assert not hasattr(app, "modes") + + mod = mutable_mock_mods_repo.get("spack-mod") + assert hasattr(mod, "modes") + assert not hasattr(mod, "workloads") + + +def test_eager_directive_execution_without_dicts(): + """Verify that directives declared with dicts=() execute eagerly at class definition.""" + executed = [] + + @DirectiveMeta.directive(dicts=()) + def custom_eager_directive(val): + def _exec(cls): + executed.append(val) + + return _exec + + class EagerApp(metaclass=DirectiveMeta): + name = "eager_app" + __module__ = "ramble.app" + custom_eager_directive("eager_ran") + + assert executed == ["eager_ran"] + + +def test_clone_nested_directive_isolation(mutable_mock_apps_repo): + """Verify that modifying nested directive dictionaries on a clone + does not mutate the original. + """ + app = mutable_mock_apps_repo.get("basic") + app.set_variables_and_variants({"workload_name": "test_wl"}, {}, None, None) + _ = app.inputs + _ = app.executables + + clone = app.clone() + # Mutate clone's inputs + clone.inputs[frozenset()]["input"]["url"] = "file:///tmp/mutated.log" + assert app.inputs[frozenset()]["input"]["url"] != "file:///tmp/mutated.log" + + # Mutate clone's executables + clone.executables[frozenset()]["foo"].template = ["mutated_cmd"] + assert app.executables[frozenset()]["foo"].template != ["mutated_cmd"] + + +def test_same_name_subclass_preferred_version_override(): + """Verify that a subclass sharing the same class name as its base + can override the preferred version. + """ + + class SharedNameApp(ExecutableApplication): + name = "shared_name_app" + __module__ = "ramble.app.base" + ramble.language.shared_language.version("1.0", preferred=True) + + # Subclass with identical class name in a different namespace + class SubclassSharedNameApp(SharedNameApp): + name = "subclass_shared_name_app" + __module__ = "ramble.app.derived" + ramble.language.shared_language.version("2.0", preferred=True) + + # Rename class attribute __name__ to mimic same class name + SubclassSharedNameApp.__name__ = "SharedNameApp" + + assert str(SharedNameApp.preferred_version.version) == "1.0" + assert str(SubclassSharedNameApp.preferred_version.version) == "2.0" + + +def test_utility_and_modifier_descriptor_scoping(mutable_mock_apps_repo, mutable_mock_mods_repo): + """Verify that utility and modifier directives do not leak descriptors onto applications.""" + import ramble.language.utility_language # noqa: F401 + + app = mutable_mock_apps_repo.get("basic") + assert not hasattr(app, "provided_executables") + assert not hasattr(app, "env_sources") + assert not hasattr(app, "env_var_modifications") + assert not hasattr(app, "modifier_conflicts") + + mod = mutable_mock_mods_repo.get("spack-mod") + assert not hasattr(mod, "provided_executables") + assert not hasattr(mod, "env_sources") + assert not hasattr(mod, "license_names") + assert not hasattr(mod, "cleanups") + + +def test_remove_directives_nested_dict(): + """Verify that directives nested inside dictionaries or sets are removed from queue.""" + + @DirectiveMeta.directive(dicts=()) + def outer_directive(**kwargs): + def _exec(cls): + pass + + return _exec + + @DirectiveMeta.directive(dicts=()) + def inner_directive(): + def _exec(cls): + pass + + return _exec + + class NestedDirectiveApp(metaclass=DirectiveMeta): + name = "nested_dir_app" + __module__ = "ramble.app" + outer_directive(mapping={"k": inner_directive()}) + + # inner_directive was passed inside a dict value, so it should have been removed + # from the execution queue, leaving only outer_directive + assert len(NestedDirectiveApp._directives_to_be_executed) == 1 + assert NestedDirectiveApp._directives_to_be_executed[0][0] == "outer_directive" + + +def test_directive_auto_registration_type_scoping(): + """Verify that new directives dynamically defined with non-shared language_type + are automatically added to _type_scoped_dicts without manual table editing. + """ + + @DirectiveMeta.directive(dicts="auto_app_dict", language_type="application") + def auto_app_directive(): + def _exec(cls): + pass + + return _exec + + @DirectiveMeta.directive(dicts="auto_mod_dict", language_type="modifier") + def auto_mod_directive(): + def _exec(cls): + pass + + return _exec + + assert "auto_app_dict" in DirectiveMeta._type_scoped_dicts["application"] + assert "auto_mod_dict" in DirectiveMeta._type_scoped_dicts["modifier"] + + class AutoApp(metaclass=DirectiveMeta): + name = "auto_app" + __module__ = "ramble.app" + _language_types = ["application", "shared"] + + class AutoMod(metaclass=DirectiveMeta): + name = "auto_mod" + __module__ = "ramble.mod" + _language_types = ["modifier", "shared"] + + # AutoApp should have auto_app_dict, but not auto_mod_dict + assert hasattr(AutoApp, "auto_app_dict") + assert not hasattr(AutoApp, "auto_mod_dict") + + # AutoMod should have auto_mod_dict, but not auto_app_dict + assert hasattr(AutoMod, "auto_mod_dict") + assert not hasattr(AutoMod, "auto_app_dict") + + +def test_class_body_scratch_variable_collision(): + """Verify that a class body local variable with a colliding name but incompatible type + does not clobber the directive dictionary initial value. + """ + + class ScratchVarApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "scratch_var_app" + __module__ = "ramble.app" + # 'workloads' is defined as a list in class body (e.g. loop accumulator), + # which collides with the 'workloads' directive dictionary (dict). + workloads = ["w1", "w2"] + for w in workloads: + ramble.language.application_language.workload(w, executables=["exe"]) + + assert isinstance(ScratchVarApp.workloads, dict) + assert "w1" in ScratchVarApp.workloads[frozenset()] + assert "w2" in ScratchVarApp.workloads[frozenset()] + + +def test_variable_and_workload_variable_multi_dict_dependencies(): + """Verify that accessing validators or object_environment_variables first + triggers execution of variable and workload_variable directives. + """ + + class VarDepsApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "var_deps_app" + __module__ = "ramble.app" + ramble.language.application_language.workload("test_wl", executables=["exe"]) + ramble.language.shared_language.variable( + "strict_var", + default="opt1", + description="Strict var", + values=["opt1", "opt2"], + strict=True, + environment_variable_name="STRICT_ENV_VAR", + ) + ramble.language.application_language.workload_variable( + "wl_strict_var", + default="v1", + description="Workload strict var", + values=["v1", "v2"], + strict=True, + workload="test_wl", + ) + + # Access validators and object_environment_variables FIRST before object_variables + validators = VarDepsApp.validators + env_vars = VarDepsApp.object_environment_variables + + assert len(validators) == 2 + assert len(env_vars) > 0 + + +def test_source_script_directive_list_init(): + """Verify that source_script initializes scripts_to_source as a list.""" + + class ScriptApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "script_app" + __module__ = "ramble.app" + ramble.language.shared_language.source_script("/path/to/my_script.sh") + + scripts = ScriptApp.scripts_to_source + assert isinstance(scripts, list) + assert len(scripts) == 1 + assert scripts[0]["path"] == "/path/to/my_script.sh" + + +def test_directive_init_values_not_clobbered(): + """Verify that registering a directive with explicit init_value=[] is not + overwritten with {} by a subsequent directive with default init_value=_UNSET. + """ + + @DirectiveMeta.directive(dicts="custom_list_dict", init_value=[]) + def first_directive(): + def _exec(cls): + cls.custom_list_dict.append("item1") + + return _exec + + @DirectiveMeta.directive(dicts=("other_dict", "custom_list_dict")) + def second_directive(): + def _exec(cls): + cls.custom_list_dict.append("item2") + + return _exec + + assert isinstance(DirectiveMeta._directive_init_values["custom_list_dict"], list) + + class ClobberTestApp(metaclass=DirectiveMeta): + name = "clobber_test_app" + __module__ = "ramble.app" + first_directive() + second_directive() + + assert ClobberTestApp.custom_list_dict == ["item1", "item2"] + + +def test_class_body_exception_prepare_isolation(): + """Verify that an exception raised halfway through a class body does not + leak queued directives or active when contexts to subsequent classes. + """ + with pytest.raises(RuntimeError, match="Intentional crash"): + + class CrashingApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "crashing_app" + __module__ = "ramble.app" + ramble.language.application_language.workload("leaked_wl", executables=["exe"]) + with ramble.language.shared_language.when("package_manager=spack"): + raise RuntimeError("Intentional crash") + + class CleanApp(metaclass=ramble.language.application_language.ApplicationMeta): + name = "clean_app" + __module__ = "ramble.app" + + assert len(CleanApp.workloads) == 0 + assert len(DirectiveMeta._when_constraints_from_context) == 0 + + +def test_eager_directive_nested_execution_depth_guard(): + """Verify that an eager directive (dicts=()) calling another directive internally + does not leak the nested directive into _directives_to_be_executed. + """ + + @DirectiveMeta.directive(dicts=()) + def eager_outer(): + def _exec(cls): + ramble.language.shared_language.tags("eager_tag")(cls) + + return _exec + + class EagerNestedApp(metaclass=DirectiveMeta): + name = "eager_nested_app" + __module__ = "ramble.app" + eager_outer() + + assert len(DirectiveMeta._directives_to_be_executed) == 0 + + +def test_partial_evaluation_rollback_on_exception(): + """Verify that if a directive raises an exception during evaluation, + initialized attributes are rolled back to _UNSET so subsequent accesses + re-raise rather than returning a corrupted dictionary. + """ + + class BrokenApp(ExecutableApplication): + name = "broken_app" + __module__ = "ramble.app" + ramble.language.shared_language.version("1.0", preferred=True) + ramble.language.shared_language.version("2.0", preferred=True) + + with pytest.raises(DirectiveError, match="already has a preferred version"): + _ = BrokenApp.known_versions + + assert BrokenApp.__dict__.get("_known_versions", _UNSET) is _UNSET + + # Second access should raise the same DirectiveError rather than returning partial state + with pytest.raises(DirectiveError, match="already has a preferred version"): + _ = BrokenApp.known_versions + + +def test_utility_base_lazy_initialization(): + """Verify UtilityBase instantiation does not eagerly evaluate its directive dictionaries.""" + UtilityBase = ramble.repository.get_base_class("utility-base") + + class LazyUtil(UtilityBase): + name = "lazy_util" + __module__ = "ramble.utility" + + util = LazyUtil("/mock/path") + for attr in [ + "env_sources", + "env_sets", + "env_prepends", + "env_appends", + "fetch_mappings", + "bootstrappable", + "missing_error_messages", + "provided_executables", + ]: + assert attr not in util.__dict__ diff --git a/lib/ramble/ramble/test/language/test_requires_utility.py b/lib/ramble/ramble/test/language/test_requires_utility.py index a66dd670f5..037f842891 100644 --- a/lib/ramble/ramble/test/language/test_requires_utility.py +++ b/lib/ramble/ramble/test/language/test_requires_utility.py @@ -9,10 +9,6 @@ import ramble.language.shared_language from ramble.language.language_base import DirectiveMeta -# Save and clear the global directives queue to isolate from other tests -_saved_queue = DirectiveMeta._directives_to_be_executed.copy() -DirectiveMeta._directives_to_be_executed = [] - class MockObject(metaclass=DirectiveMeta): name = "mock_obj" @@ -59,9 +55,6 @@ class MockObject(metaclass=DirectiveMeta): ) -DirectiveMeta._directives_to_be_executed = _saved_queue - - def test_requires_utility_directive_parsing(): obj = MockObject() when_list = ["mock_when=True"] diff --git a/lib/ramble/ramble/test/mirror.py b/lib/ramble/ramble/test/mirror.py index b1e470b34a..b5b545b52f 100644 --- a/lib/ramble/ramble/test/mirror.py +++ b/lib/ramble/ramble/test/mirror.py @@ -80,39 +80,37 @@ def test_mirror_str_and_repr(): # Create an archive for the test input, with the correct file name -def create_archive(archive_dir, app_class): +def create_archive(archive_dir, app_cls): tar = spack.util.executable.which("tar", required=True) - app_class._inputs_and_fetchers() - - for input_name, conf in app_class._input_fetchers.items(): - if conf["expand"]: - archive_dir.ensure(input_name, dir=True) - archive_name = os.path.basename(conf["fetcher"].url) - test_file_path = str(archive_dir.join(input_name, "input-file")) - with open(test_file_path, "w+", encoding="utf-8") as f: - f.write("Input File\n") - - with archive_dir.as_cwd(): - tar("-czf", archive_name, input_name) - with open(archive_name, "rb") as f: - bytes = f.read() - conf["fetcher"].digest = hashlib.sha256(bytes).hexdigest() - app_class.inputs[_FS][conf["input_name"]]["sha256"] = conf["fetcher"].digest - else: - with open(input_name, "w+", encoding="utf-8") as f: - f.write("Input file\n") - - with open(input_name, "rb") as f: - bytes = f.read() - conf["fetcher"].digest = hashlib.sha256(bytes).hexdigest() - app_class.inputs[_FS][conf["input_name"]]["sha256"] = conf["fetcher"].digest - - -def check_mirror(mirror_path, app_name, app_class): - app_class._inputs_and_fetchers() - - for input_name, conf in app_class._input_fetchers.items(): + for inputs_dict in app_cls.inputs.values(): + for input_name, conf in inputs_dict.items(): + expand = conf.get("expand", True) + url = conf["url"] + if expand: + archive_dir.ensure(input_name, dir=True) + archive_name = os.path.basename(url) + test_file_path = str(archive_dir.join(input_name, "input-file")) + with open(test_file_path, "w+", encoding="utf-8") as f: + f.write("Input File\n") + + with archive_dir.as_cwd(): + tar("-czf", archive_name, input_name) + with open(archive_name, "rb") as f: + conf["sha256"] = hashlib.sha256(f.read()).hexdigest() + else: + filename = os.path.basename(url) + with open(filename, "w+", encoding="utf-8") as f: + f.write("Input file\n") + + with open(filename, "rb") as f: + conf["sha256"] = hashlib.sha256(f.read()).hexdigest() + + +def check_mirror(mirror_path, app_name, app_inst): + app_inst._inputs_and_fetchers() + + for input_name, conf in app_inst._input_fetchers.items(): test_name = f"{input_name}" fetcher = conf["fetcher"] if fetcher.extension: @@ -153,9 +151,8 @@ def test_mirror_create(tmpdir, mutable_mock_workspace_path, app_name, tmpdir_fac with archive_dir.as_cwd(): app_type = ramble.repository.ObjectTypes.applications - app_class = ramble.repository.paths[app_type].get_obj_class(app_name)("test") - app_class.set_variables_and_variants({"workload_name": "test"}, {}, None, None) - create_archive(archive_dir, app_class) + app_cls = ramble.repository.paths[app_type].get_obj_class(app_name) + create_archive(archive_dir, app_cls) # Create workspace ws_name = f"workspace-mirror-{app_name}" @@ -169,4 +166,6 @@ def test_mirror_create(tmpdir, mutable_mock_workspace_path, app_name, tmpdir_fac mirror_pipeline = mirror_pipeline_cls(workspace, filters, mirror_path=str(mirror_dir)) mirror_pipeline.run() - check_mirror(str(mirror_dir), app_name, app_class) + app_inst = app_cls("test") + app_inst.set_variables_and_variants({"workload_name": "test"}, {}, None, None) + check_mirror(str(mirror_dir), app_name, app_inst) diff --git a/lib/ramble/ramble/util/class_attributes.py b/lib/ramble/ramble/util/class_attributes.py deleted file mode 100644 index a688c843b5..0000000000 --- a/lib/ramble/ramble/util/class_attributes.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2022-2026 The Ramble Authors -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - - -def convert_class_attributes(obj): - """Convert class attributes defined from directives to instance attributes - Class attributes that are valid for conversion are stored in the _directive_names - attribute. - - Args: - obj (object): Input object instance to convert attributes in - """ - - if hasattr(obj, "_directive_names"): - var_set = vars(obj) - for attr in obj._directive_names: - if attr not in var_set and hasattr(obj, attr): - val = getattr(obj, attr) - if hasattr(val, "copy"): - inst_val = val.copy() - else: - inst_val = val - setattr(obj, attr, inst_val) diff --git a/lib/ramble/ramble/util/directives.py b/lib/ramble/ramble/util/directives.py index 89ffa59e5a..5d2ad235d9 100644 --- a/lib/ramble/ramble/util/directives.py +++ b/lib/ramble/ramble/util/directives.py @@ -12,12 +12,19 @@ def define_directive_methods_on_class(cls): Wrap each directive, and inject it into this class as a method. """ - if not hasattr(cls, "_directive_classes") or not hasattr(cls, "_directive_functions"): + if not hasattr(cls, "_directive_functions"): return - lang_classes = getattr(cls, "_language_classes", []) - for directive, directive_class in cls._directive_classes.items(): - if directive_class in lang_classes and not hasattr(cls, directive): + lang_types = set(getattr(cls, "_language_types", [])) | set( + getattr(cls, "_language_classes", []) + ) + directive_types = getattr(cls, "_directive_types", {}) + directive_classes = getattr(cls, "_directive_classes", {}) + + for directive in cls._directive_functions: + d_type = directive_types.get(directive) + d_cls = directive_classes.get(directive) + if (d_type in lang_types or d_cls in lang_types) and not hasattr(cls, directive): setattr(cls, directive, wrap_named_directive_class_level(directive)) @@ -28,6 +35,12 @@ def wrap_named_directive_class_level(name): """ def _execute_directive(self, *args, directive_name=name, **kwargs): - self._directive_functions[directive_name](*args, **kwargs)(self) + import ramble.language.language_base + + ramble.language.language_base.DirectiveMeta._executing_directives_depth += 1 + try: + self._directive_functions[directive_name](*args, **kwargs)(self) + finally: + ramble.language.language_base.DirectiveMeta._executing_directives_depth -= 1 return _execute_directive diff --git a/var/ramble/repos/builtin/base_classes/application-base/base_class.py b/var/ramble/repos/builtin/base_classes/application-base/base_class.py index b3f9fe1b81..1a6d55261a 100644 --- a/var/ramble/repos/builtin/base_classes/application-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/application-base/base_class.py @@ -33,7 +33,6 @@ import ramble.repository import ramble.stage import ramble.success_criteria -import ramble.util.class_attributes import ramble.util.colors as rucolor import ramble.util.env import ramble.util.executable @@ -52,9 +51,8 @@ ObjectValidationError, ) from ramble.experiment_result import ExperimentResult, ExperimentStatus -from ramble.language.application_language import ApplicationMeta +from ramble.language.language_base import DirectiveMeta from ramble.language.shared_language import ( - SharedMeta, archive_pattern, register_builtin, register_phase, @@ -152,7 +150,7 @@ def _get_phase_func_wrapper(workspace, phase_func, phase_name): return profiler(phase_func) -class ApplicationBase(ObjectMixin, metaclass=ApplicationMeta): +class ApplicationBase(ObjectMixin, metaclass=DirectiveMeta): _mro_obj_type_cache = {} name = "application-base" origin_type = "application" @@ -171,7 +169,8 @@ class ApplicationBase(ObjectMixin, metaclass=ApplicationMeta): "execute", "logs", ] - _language_classes = [ApplicationMeta, SharedMeta] + _language_types = ["application", "shared"] + _language_classes = _language_types variant( "inject_modifiers_from_directives", @@ -192,8 +191,6 @@ class ApplicationBase(ObjectMixin, metaclass=ApplicationMeta): def __init__(self, file_path): super().__init__() - ramble.util.class_attributes.convert_class_attributes(self) - self.object_variants = ramble.variants.VariantSet() for var_args in self.class_variants.values(): self.object_variants.default_variant(**var_args) @@ -222,7 +219,7 @@ def __init__(self, file_path): self._vars_are_expanded = False self.expander = None - self._formatted_executables = {} + self._context_formatted_executables = {} self.variables = None self.variants = None self._active_workload = None @@ -305,8 +302,8 @@ def clone(self): new_clone = type(self)(self._file_path) self.has_generated_experiments = True - if self.known_versions: - new_clone.known_versions = self.known_versions.copy() + self._copy_evaluated_directives(new_clone) + clone_variables = {} if not self.variables else self.variables clone_variants = {} if not self.variants else self.variants new_clone.set_variables_and_variants( @@ -319,13 +316,11 @@ def clone(self): new_clone.set_env_variable_sets(self._env_variable_sets.copy()) if self.internals: new_clone.set_internals(self.internals.copy()) - if self._formatted_executables: + if self._context_formatted_executables: new_clone.set_formatted_executables( - self._formatted_executables.copy() + self._context_formatted_executables.copy() ) - new_clone.workloads = copy.deepcopy(self.workloads) - new_clone.inputs = copy.deepcopy(self.inputs) new_clone.custom_executables = self.custom_executables.copy() new_clone.keywords = ramble.keywords.keywords.copy() new_clone.set_template(False) @@ -744,7 +739,7 @@ def set_variables_and_variants( version_number=maybe_version, description=self.expander.application_spec, ) - elif hasattr(self, "preferred_version"): + elif self.preferred_version is not None: super().set_version( version=self.preferred_version, description=self.expander.application_spec, @@ -1096,7 +1091,7 @@ def set_tags(self, tags): def set_formatted_executables(self, formatted_executables): """Set formatted executables for this instance""" - self._formatted_executables = formatted_executables.copy() + self._context_formatted_executables = formatted_executables.copy() def has_tags(self, tags): """Check if this instance has provided tags. @@ -2469,7 +2464,9 @@ def _define_formatted_executables(self): self.variables[self.keywords.unformatted_command_without_logs] = ( "\n".join(self._command_list_without_logs) ) - formatted_exec_groups = [{frozenset(): self._formatted_executables}] + formatted_exec_groups = [ + {frozenset(): self._context_formatted_executables} + ] objs_to_extract = [self, self.workflow_manager, self.package_manager] diff --git a/var/ramble/repos/builtin/base_classes/modifier-base/base_class.py b/var/ramble/repos/builtin/base_classes/modifier-base/base_class.py index c04102440f..b53a025239 100644 --- a/var/ramble/repos/builtin/base_classes/modifier-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/modifier-base/base_class.py @@ -11,19 +11,17 @@ import re import ramble.repository -import ramble.util.class_attributes import ramble.variants from ramble.error import ( ConflictingModifiersError, InvalidModeError, ModifierError, ) +from ramble.language.language_base import DirectiveMeta from ramble.language.modifier_language import ( - ModifierMeta, mode, modifier_conflict, ) -from ramble.language.shared_language import SharedMeta from ramble.util.conflicts import MODIFIER_CONFLICT from ramble.util.logger import logger from ramble.util.naming import NS_SEPARATOR @@ -31,14 +29,15 @@ ObjectMixin = ramble.repository.get_base_class("object-mixin") -class ModifierBase(ObjectMixin, metaclass=ModifierMeta): +class ModifierBase(ObjectMixin, metaclass=DirectiveMeta): name = "modifier-base" origin_type = "modifier" _builtin_name = NS_SEPARATOR.join( ("modifier_builtin", "{obj_name}", "{name}") ) _mod_prefix_builtin = f"modifier_builtin{NS_SEPARATOR}" - _language_classes = [ModifierMeta, SharedMeta] + _language_types = ["modifier", "shared"] + _language_classes = _language_types pipelines = [ "analyze", "archive", @@ -63,8 +62,6 @@ def __init__(self, file_path): for var_args in self.class_variants.values(): self.object_variants.default_variant(**var_args) - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self._on_executables = ["*"] self.expander = None @@ -96,8 +93,8 @@ def set_usage_mode(self, mode): """ if mode: self._usage_mode = mode - elif hasattr(self, "_default_usage_mode"): - self._usage_mode = self._default_usage_mode + elif getattr(self, "default_usage_mode", None) is not None: + self._usage_mode = self.default_usage_mode if len(logger.log_stack) >= 1: logger.msg( f" Using default usage mode {self._usage_mode} on modifier {self.name}" diff --git a/var/ramble/repos/builtin/base_classes/object-mixin/base_class.py b/var/ramble/repos/builtin/base_classes/object-mixin/base_class.py index 2d516c0043..a2321e58bd 100644 --- a/var/ramble/repos/builtin/base_classes/object-mixin/base_class.py +++ b/var/ramble/repos/builtin/base_classes/object-mixin/base_class.py @@ -8,7 +8,7 @@ import functools import os from html import escape -from typing import List, Optional +from typing import Any, List, Optional import ramble.config from ramble.definitions.versions import ObjectVersion @@ -58,11 +58,55 @@ def name(self): def scoped_name(self): return f"{self.origin_type}::{self.name}" + @property + def preferred_version(self) -> Optional[ObjectVersion]: + for ver in getattr(self, "known_versions", {}).values(): + if getattr(ver, "preferred", False): + return ver + return None + + @preferred_version.setter + def preferred_version(self, value: Optional[Any]): + if not hasattr(self, "known_versions"): + return + for ver in self.known_versions.values(): + if hasattr(ver, "preferred"): + ver.preferred = False + if value is not None: + try: + value.preferred = True + except (AttributeError, TypeError): + pass + ver_key = ( + getattr(value, "version_number", None) + or getattr(value, "version", None) + or str(value) + ) + self.known_versions[ver_key] = value + + def _copy_evaluated_directives(self, target): + """Copy all evaluated directive dictionaries from self to target.""" + from ramble.language.language_base import _UNSET, _copy_directive_value + + directive_dicts = getattr(self, "_directive_dict_names", set()) + + for dict_name in directive_dicts.intersection(self.__dict__): + val = self.__dict__[dict_name] + if val is _UNSET: + continue + if ( + dict_name in target.__dict__ + and target.__dict__[dict_name] == val + ): + continue + target.__dict__[dict_name] = _copy_directive_value(val) + def copy(self): """Generic copy method for Ramble objects.""" new_copy = type(self)(self._file_path) if hasattr(self, "_verbosity"): new_copy._verbosity = self._verbosity + self._copy_evaluated_directives(new_copy) return new_copy def all_pipeline_phases(self, pipeline): diff --git a/var/ramble/repos/builtin/base_classes/package-manager-base/base_class.py b/var/ramble/repos/builtin/base_classes/package-manager-base/base_class.py index 1d9f0239e5..60bf5feee0 100644 --- a/var/ramble/repos/builtin/base_classes/package-manager-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/package-manager-base/base_class.py @@ -14,10 +14,8 @@ import ramble.definitions.families import ramble.repository -import ramble.util.class_attributes -import ramble.variants -from ramble.language.package_manager_language import PackageManagerMeta -from ramble.language.shared_language import SharedMeta, register_phase +from ramble.language.language_base import DirectiveMeta +from ramble.language.shared_language import register_phase from ramble.software_environments import ( RambleSoftwareEnvironmentError, TemplatePackage, @@ -28,12 +26,13 @@ ObjectMixin = ramble.repository.get_base_class("object-mixin") -class PackageManagerBase(ObjectMixin, metaclass=PackageManagerMeta): +class PackageManagerBase(ObjectMixin, metaclass=DirectiveMeta): origin_type = "package_manager" _builtin_name = NS_SEPARATOR.join( ("package_manager_builtin", "{obj_name}", "{name}") ) - _language_classes = [PackageManagerMeta, SharedMeta] + _language_types = ["package_manager", "shared"] + _language_classes = _language_types pipelines = [ "analyze", "archive", @@ -67,8 +66,6 @@ def __init__(self, file_path): self.origin_type, list(self.class_families) ) - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self.app_inst = None diff --git a/var/ramble/repos/builtin/base_classes/platform-base/base_class.py b/var/ramble/repos/builtin/base_classes/platform-base/base_class.py index 0aa855dd38..1cd8728695 100644 --- a/var/ramble/repos/builtin/base_classes/platform-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/platform-base/base_class.py @@ -8,11 +8,9 @@ """Define base classes for platform definitions""" import ramble.definitions.families -import ramble.util.class_attributes import ramble.variants -from ramble.language.platform_language import PlatformMeta +from ramble.language.language_base import DirectiveMeta from ramble.language.shared_language import ( - SharedMeta, register_validator, required_variable, variant, @@ -23,13 +21,14 @@ ObjectMixin = ramble.repository.get_base_class("object-mixin") -class PlatformBase(ObjectMixin, metaclass=PlatformMeta): +class PlatformBase(ObjectMixin, metaclass=DirectiveMeta): name = None origin_type = "platform" _builtin_name = NS_SEPARATOR.join( ("platform_builtin", "{obj_name}", "{name}") ) - _language_classes = [PlatformMeta, SharedMeta] + _language_types = ["platform", "shared"] + _language_classes = _language_types variant( "accelerator", @@ -102,8 +101,6 @@ def __init__(self, file_path): self.origin_type, list(self.class_families.keys()) ) - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self.object_variants.default_variant( diff --git a/var/ramble/repos/builtin/base_classes/system-base/base_class.py b/var/ramble/repos/builtin/base_classes/system-base/base_class.py index 2a035bcf4d..3f3d40373c 100644 --- a/var/ramble/repos/builtin/base_classes/system-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/system-base/base_class.py @@ -8,33 +8,27 @@ """Define base classes for system definitions""" import ramble.definitions.families -import ramble.util.class_attributes import ramble.variants +from ramble.language.language_base import DirectiveMeta from ramble.language.shared_language import ( - SharedMeta, register_validator, required_variable, variant, when, ) -from ramble.language.system_language import SystemMeta from ramble.util.naming import NS_SEPARATOR ObjectMixin = ramble.repository.get_base_class("object-mixin") -class SystemBase(ObjectMixin, metaclass=SystemMeta): +class SystemBase(ObjectMixin, metaclass=DirectiveMeta): name = None origin_type = "system" _builtin_name = NS_SEPARATOR.join( ("system_builtin", "{obj_name}", "{name}") ) - _language_classes = [SystemMeta, SharedMeta] - - system_default_platform = None - system_default_workflow_manager = None - system_default_package_manager = None - system_available_platforms = [] + _language_types = ["system", "shared"] + _language_classes = _language_types variant( "validate_system", @@ -72,8 +66,6 @@ def __init__(self, file_path): self.origin_type, list(self.class_families.keys()) ) - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self.object_variants.default_variant( diff --git a/var/ramble/repos/builtin/base_classes/utility-base/base_class.py b/var/ramble/repos/builtin/base_classes/utility-base/base_class.py index b01955a0ff..2f9439d482 100644 --- a/var/ramble/repos/builtin/base_classes/utility-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/utility-base/base_class.py @@ -9,22 +9,20 @@ import os import ramble.repository -import ramble.util.class_attributes -import ramble.variants -from ramble.language.shared_language import SharedMeta -from ramble.language.utility_language import UtilityMeta +from ramble.language.language_base import DirectiveMeta from ramble.util.logger import logger from ramble.util.naming import NS_SEPARATOR ObjectMixin = ramble.repository.get_base_class("object-mixin") -class UtilityBase(ObjectMixin, metaclass=UtilityMeta): +class UtilityBase(ObjectMixin, metaclass=DirectiveMeta): origin_type = "utility" _builtin_name = NS_SEPARATOR.join( ("utility_builtin", "{obj_name}", "{name}") ) - _language_classes = [UtilityMeta, SharedMeta] + _language_types = ["utility", "shared"] + _language_classes = _language_types pipelines = [ "setup", ] @@ -38,19 +36,6 @@ def __init__(self, file_path): for var_args in self.class_variants.values(): self.object_variants.default_variant(**var_args) - self.env_sources = getattr(self, "env_sources", {}) - self.env_sets = getattr(self, "env_sets", {}) - self.env_prepends = getattr(self, "env_prepends", {}) - self.env_appends = getattr(self, "env_appends", {}) - self.fetch_mappings = getattr(self, "fetch_mappings", {}) - self.bootstrappable = getattr(self, "bootstrappable", {}) - self.missing_error_messages = getattr( - self, "missing_error_messages", {} - ) - self.provided_executables = getattr(self, "provided_executables", {}) - - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self.keywords = None diff --git a/var/ramble/repos/builtin/base_classes/workflow-manager-base/base_class.py b/var/ramble/repos/builtin/base_classes/workflow-manager-base/base_class.py index d8a83af2ae..34b4aa8c89 100644 --- a/var/ramble/repos/builtin/base_classes/workflow-manager-base/base_class.py +++ b/var/ramble/repos/builtin/base_classes/workflow-manager-base/base_class.py @@ -11,12 +11,10 @@ from typing import Collection, Iterator import ramble.definitions.families -import ramble.util.class_attributes import ramble.variants from ramble.expander import ExpanderError -from ramble.language.shared_language import SharedMeta +from ramble.language.language_base import DirectiveMeta from ramble.language.workflow_manager_language import ( - WorkflowManagerMeta, workflow_manager_variable, ) from ramble.util.naming import NS_SEPARATOR @@ -25,12 +23,13 @@ ObjectMixin = ramble.repository.get_base_class("object-mixin") -class WorkflowManagerBase(ObjectMixin, metaclass=WorkflowManagerMeta): +class WorkflowManagerBase(ObjectMixin, metaclass=DirectiveMeta): origin_type = "workflow_manager" _builtin_name = NS_SEPARATOR.join( ("workflow_manager_builtin", "{obj_name}", "{name}") ) - _language_classes = [WorkflowManagerMeta, SharedMeta] + _language_types = ["workflow_manager", "shared"] + _language_classes = _language_types pipelines = [ "analyze", "setup", @@ -74,8 +73,6 @@ def __init__(self, file_path): self.origin_type, list(self.class_families.keys()) ) - ramble.util.class_attributes.convert_class_attributes(self) - self._file_path = file_path self.object_variants.default_variant(