diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 2f67e38e1..fcdde1983 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -17,19 +17,18 @@ import os from pathlib import Path import sys -from typing import List, Optional, Iterable, Union +from typing import cast, Optional, Iterable, Union from fab.api import (ArtefactSet, BuildConfig, Category, Exclude, grab_folder, - Include, input_to_output_fpath, preprocess_x90, psyclone, - step, SuffixFilter) + Include, input_to_output_fpath, Linker, preprocess_x90, + psyclone, psyclone_transmute, step, SuffixFilter) from fab.fab_base.fab_base import FabBase from configurator import configurator from templaterator import Templaterator +from psyclone_control import PsycloneControl, PsycloneInfo -# Add a logger and connect it to stdout. -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) +logger = logging.getLogger("fab") class LFRicBase(FabBase): @@ -46,15 +45,20 @@ class LFRicBase(FabBase): # pylint: disable=too-many-instance-attributes def __init__(self, name: str, app_dir: Path, - root_symbol: Optional[Union[List[str], str]] = None + root_symbol: Optional[Union[list[str], str]] = None, ): self._app_dir = app_dir + this_file = Path(__file__) + # The root directory of the LFRic Core + self._lfric_core_root = this_file.parents[1] + # List of all precision preprocessor symbols and their default. # Used to add corresponding command line options, and then to define # the preprocessor definitions. Note that precision_other - # becomes RDEF. + # becomes RDEF. Must be defined before calling super().__init__ + # (since it is required when defining command line options). self._all_precisions = [("precision_other", "64"), ("R_SOLVER_PRECISION", "32"), ("R_TRAN_PRECISION", "64"), @@ -62,10 +66,6 @@ def __init__(self, name: str, super().__init__(name) - this_file = Path(__file__) - # The root directory of the LFRic Core - self._lfric_core_root = this_file.parents[1] - # If the user wants to overwrite the default root symbol (which # is `name`): if root_symbol: @@ -79,6 +79,7 @@ def __init__(self, name: str, mpi=self.config.mpi, openmp=self.config.openmp, enforce_fortran_linker=True) + linker = cast(Linker, linker) try: linker.get_lib_flags("netcdf") except RuntimeError as err: @@ -86,6 +87,35 @@ def __init__(self, name: str, f"has no NetCDF library setting defined. Aborting.") raise RuntimeError(msg) from err + # Store the PSyclone control information + # The paths used when identifying file-specific scripts. The matching + # base path is removed from the file name to get a relative name + # which is used in matching. + base_paths = [self.config.source_root, + self.config.build_output] + script_root = (self.config.source_root / "optimisation" / + f"{self.site}-{self.platform}") + self._psyclone_control = PsycloneControl(script_root=script_root, + base_paths=base_paths) + + # If the PSyclone step is running, this stores the currently + # executed PSyclone information. This is used in the + # get_transformation_script function (to map the special return + # values like NO_SCRIPT back for the PSyclone tool). + self._current_psyclone_info: Optional[PsycloneInfo] = None + + if self.args.psyclone_control: + control_list = [Path(i) for i in self.args.psyclone_control] + else: + # This default rule implements the "file-specific if exists, + # otherwise global.py" rule. + control_list = [Path(self.lfric_core_root / "lfric_build" / + "psyclone_control.yaml")] + for psy_info_file in control_list: + logger.info(f"Reading PSyclone configuration file " + f"'{psy_info_file}'.") + self._psyclone_control.read(Path(psy_info_file)) + @property def app_dir(self) -> Path: """ @@ -100,6 +130,17 @@ def lfric_core_root(self) -> Path: ''' return self._lfric_core_root + def add_python_search_path(self, path: Union[str, Path]) -> None: + ''' + Adds the specified path to the list of python search paths, which + will be added before PSyclone is executed. This allows an application + to add additional search paths required for the application (e.g. + the location of transmute scripts). + + :param path: the search path to add. + ''' + self._add_python_paths.append(str(path)) + def define_command_line_options( self, parser: Optional[argparse.ArgumentParser] = None @@ -119,6 +160,11 @@ def define_command_line_options( '--no-xios', action="store_true", default=False, help="Disable compilation with XIOS.") + parser.add_argument( + '--psyclone-control', action="append", + help="PSyclone configuration files, controlling when to " + "run the various PSyclone phases.") + # Precision related command line arguments # ---------------------------------------- group = parser.add_argument_group( @@ -154,21 +200,6 @@ def handle_command_line_options(self, "command line.") sys.exit(-1) - def setup_site_specific_location(self): - ''' - This method adds the required directories for site-specific - configurations to the Python search path. We want to add the - directory where this lfric_base class is located, and not the - directory in which the application script is (which is what - baf base would set up). - ''' - this_dir = Path(__file__).parent - # We need to add the 'site_specific' directory to the path, so - # each config can import from 'default' (instead of having to - # use 'site_specific.default', which would hard-code the name - # `site_specific` in more scripts). - sys.path.insert(0, str(this_dir / "site_specific")) - def define_preprocessor_flags_step(self) -> None: ''' This method overwrites the base class define_preprocessor_flags. @@ -178,7 +209,7 @@ def define_preprocessor_flags_step(self) -> None: - Use of XIOS (if not disabled using --no-xios command line option) - Disabling MPI (if disabled using --no-mpi) ''' - preprocessor_flags: List[str] = [] + preprocessor_flags: list[str] = [] # Check all required precision defines for prec_name, _ in self._all_precisions: @@ -198,7 +229,7 @@ def define_preprocessor_flags_step(self) -> None: self.add_preprocessor_flags(preprocessor_flags) - def get_linker_flags(self) -> List[str]: + def get_linker_flags(self) -> list[str]: ''' This method overwrites the base class get_linker_flags. It passes the libraries that LFRic uses to the linker. Currently, these libraries @@ -213,8 +244,7 @@ def grab_files_step(self) -> None: ''' This method overwrites the base class grab_files_step. It includes all the LFRic core directories that are commonly required for building - LFRic applications. It also grabs the psydata directory for profiling, - if required. + LFRic applications. It also grabs optimisation scripts. ''' dirs = ['infrastructure/source/', 'components/driver/source/', @@ -232,6 +262,10 @@ def grab_files_step(self) -> None: grab_folder(self.config, src=self.lfric_core_root / "etc", dst_label='psyclone_config') + # Copy the optimisation scripts into a separate directory + grab_folder(self.config, src=self.app_dir / 'optimisation', + dst_label='optimisation') + def find_source_files_step( self, path_filters: Optional[Iterable[Union[Exclude, Include]]] = None @@ -356,7 +390,8 @@ def preprocess_x90_step(self) -> None: def psyclone_step( self, ignore_dependencies: Optional[Iterable[str]] = None, - additional_parameters: Optional[list[str]] = None + additional_parameters: Optional[list[str]] = None, + kernel_roots: Optional[list[Path]] = None, ) -> None: ''' This method runs Fab's psyclone. It first sets the psyclone @@ -367,27 +402,81 @@ def psyclone_step( got through calling `get_transformation_script`, the api, and the additional psyclone command line arguments. - :param ignore_dependencies: + :param ignore_dependencies: Third party Fortran module names in + USE statements, 'DEPENDS ON' files and modules to be ignored. :param additional_parameters: optional additional parameter for the PSyclone. + :param kernel_roots: + Folders containing kernel files. Must be part of the analysed + source code. ''' + logger.info(f"PSyclone control file\n" + f"# -------------------\n" + f"{self._psyclone_control.to_yaml()}\n" + f"# -------------------\n") + + kernel_roots = kernel_roots or [] psyclone_cli_args = ["--config", self.get_psyclone_config()] if additional_parameters: psyclone_cli_args.extend(additional_parameters) + add_python_paths = ":".join(str(i) for i in self._add_python_paths) # To avoid impacting other code, store the original search path + # We have to modify PYTHONPATH (and not sys.path), since PSyclone + # is run in its own shell (i.e. it inherits PYTHONPATH, but not + # sys.path). orig_pythonpath = os.environ.get("PYTHONPATH", "") - add_python_paths = ":".join(str(i) for i in self._add_python_paths) - os.environ["PYTHONPATH"] = (f"{add_python_paths}:" - f"{orig_pythonpath}") - psyclone(self.config, kernel_roots=[(self.config.build_output / - "kernel")], - transformation_script=self.get_transformation_script, - api="lfric", - cli_args=psyclone_cli_args, - ignore_dependencies=ignore_dependencies) - # Reset PYTHONPATH - os.environ["PYTHONPATH"] = orig_pythonpath + + for phase in self._psyclone_control.all_phases: + psyclone_info = self._psyclone_control.get_info(phase) + logger.info(f"Running PSyclone phase: {psyclone_info.comment}.") + # Save the info for the get transformation script function + self._current_psyclone_info = psyclone_info + # Add various paths: optimisation/site-platform/transmute + # (=opt_path) is required for some transmute scripts that + # import helper functions. Adding add_python_paths is + # required for other, shared PSyclone scripts + os.environ["PYTHONPATH"] = (f"{psyclone_info.opt_path}:" + f"{add_python_paths}:" + f"{orig_pythonpath}") + if psyclone_info.api: + psyclone(self.config, + kernel_roots=(kernel_roots + + [self.config.build_output / "kernel"]), + transformation_script=self.get_transformation_script, + api=psyclone_info.api, + cli_args=psyclone_cli_args, + ignore_dependencies=ignore_dependencies) + else: + self._psyclone_transmute(psyclone_info, + psyclone_cli_args) + # Reset PYTHONPATH + os.environ["PYTHONPATH"] = orig_pythonpath + # Reset to None + self._current_psyclone_info = None + + def _psyclone_transmute(self, + psyclone_info: PsycloneInfo, + psyclone_cli_args: list[str], + ): + f90_files: list[Path] = [] + af_store = self.config.artefact_store + for file in af_store[ArtefactSet.FORTRAN_COMPILER_FILES]: + script = psyclone_info.get_script(file) + if script != PsycloneInfo.RESULT_EXCLUDE: + f90_files.append(file) + + # Don't use a suffix, meaning the original source files will be + # overwritten. Otherwise, PSyclone might find two files with the + # same kernels (once transmuted, one original) and abort. + psyclone_transmute( + self.config, + fortran_files=f90_files, + transformation_script=self.get_transformation_script, + cli_args=psyclone_cli_args, + suffix="", + artefact_set=ArtefactSet.FORTRAN_COMPILER_FILES + ) def get_psyclone_config(self) -> str: ''' @@ -404,40 +493,24 @@ def get_transformation_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: ''' This method returns the path to the transformation script that PSyclone - will use for each x90 file. It first checks if there is a specific - transformation script for the x90 file. If not, it will see whether a - global transformation script can be used. + will use for each x90 file. It uses the get_script method of the + current PsycloneInfo object, but maps the special return values + "NO_SCRIPT". :param fpath: the path to the file being processed. :param config: the FAB BuildConfig instance. :returns: the transformation script to be used by PSyclone. + + :raises ValueError: If the current psyclone_info result indicates + that the current file should be be run through PSyclone. ''' - # Newer LFRic versions have a psykal directory - optimisation_path = (config.source_root / "optimisation" / - f"{self.site}-{self.platform}" / "psykal") - relative_path = None - # The source file might be either in build_output (e.g. a preprocessed - # .X90 file), or still in source (.x90 file). Check if the file - # is in one of the two sub-trees, and use the relative path to - # check if there is a file-specific optimisation script - for base_path in [config.source_root, config.build_output]: - try: - relative_path = fpath.relative_to(base_path) - except ValueError: - # The file is not under the `base_path` - keep on checking - pass - - if relative_path: - # The file was under either source or build. Check if there - # is a file-specific optimisation script: - local_transformation_script = (optimisation_path / - (relative_path.with_suffix('.py'))) - if local_transformation_script.exists(): - return local_transformation_script - - # No file-specific optimisation script found. Check for global.py: - global_transformation_script = optimisation_path / 'global.py' - if global_transformation_script.exists(): - return global_transformation_script - return None + script = self._current_psyclone_info.get_script(fpath) + + if script == PsycloneInfo.RESULT_NO_SCRIPT: + return None + if script == PsycloneInfo.RESULT_EXCLUDE: + raise ValueError("PSyclone transformation script returned " + "'PsycloneInfo.RESULT_EXCLUDE', which should " + "not happen.") + return script diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py new file mode 100755 index 000000000..ce5710ff9 --- /dev/null +++ b/lfric_build/psyclone_control.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 + +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +''' +This module reads in a psyclone_info.yaml file. +''' +from pathlib import Path +from typing import Optional +import yaml + + +class PsycloneInfo: + """ + This class stores the set of rules and settings for one specific PSyclone + phase. + + :param name: the name of this phase. + :param fab_base: the application script derived from FabBase. Required to + get access to the build config for paths, and the selected site and + platform. + """ + + # Define a new 'script paths' to indicate special results. Use : in the + # name, which is typically not a valid name (at the start) + RESULT_EXCLUDE = Path("::RESULT_EXCLUDE") + RESULT_NO_SCRIPT = Path("::RESULT_NO_SCRIPT") + + FILE_SPECIFIC = "file_specific" + EXCLUDE = "exclude" + NO_SCRIPT = "no_script" + + def __init__(self, name: str, + base_paths: list[Path], + script_root: Path) -> None: + self._base_paths = base_paths + self._script_root = script_root + # This will be initialised/updated each time when reading an info file. + self._opt_path = Path() + self._relative_script_dir: str = "" + self._name: str = name + self._comment: str = "" + self._api: str = "" + self._rules: list[tuple[str, list[str]]] = [] + + @property + def name(self) -> str: + """ + :returns: the name of this phas. + """ + return self._name + + @property + def comment(self) -> str: + """ + :returns: the comment for this phase. + """ + return self._comment + + @property + def api(self) -> str: + """ + :returns: the PSyclone command line options. + """ + return self._api + + @property + def opt_path(self) -> Path: + """ + :return: the optimisation root as absolute path (including site- + and platform-specific settings, and subdirectory, e.g. psykal + or transmute). + """ + return self._opt_path + + def update(self, info: dict[str, str]) -> None: + """ + Update this PSyclone information with data taken from a + PSyclone info yaml file. This function is used to initially + read in a first specification, or update a specification based + on an additional file being read in later. + + :param info: the yaml information taken from a PSyclone info file. + """ + for rule in info: + if rule == "comment": + self._comment = info["comment"] + elif rule == "api": + self._api = info["api"] + elif rule == "script_dir": + self._relative_script_dir = info["script_dir"] + else: + self._read_rule(rule, info[rule]) + + # Store the potentially updated optimisation root path, i.e. the site- + # and platform-specific location, followed by a script dir (typically + # transmute or psykal). This path is used in a few places. + self._opt_path = self._script_root / self._relative_script_dir + + def _read_rule(self, rule: str, file_list: str) -> None: + """ + Parses a single rule from the yaml file. It especially handles + various way a '*' can be specified in a yaml file. + + :param rule: the name of the rule (typically the script name, or + special term like `exclude` or `file_specific`). + :param file_list: the list of files to which to apply the rule to. + """ + # Support '*', which is a reserved character in yaml and needs to + # be escaped or quoted. + if file_list in ["\\*", "'*'", '"*"']: + file_list = "*" + self._rules.append((rule, file_list.split())) + + def get_yaml_dict(self) -> dict[str, list[str]]: + """ + :returns: returns ths data in this object in a dictionary, suitable + to write them back as a yaml file. + """ + + yaml_dict = {"comment": self.comment, + "api": self.api, + "script_dir": self._relative_script_dir} + + for rule, file_list in self._rules: + yaml_dict[rule] = " ".join(file_list) + + return yaml_dict + + def file_specific_script(self, fpath: Path) -> Optional[Path]: + """ + Searches for a file-specific optimisation script. It will search + both under the source and the build directories of the project + directory. + + :param fpath: the file path of the Fortran file. + :returns: the path of the file-specific optimisation script, or + None if no such file exists. + """ + relative_path = None + # The source file might be either in build_output (e.g. a preprocessed + # .X90 file), or still in source (.x90 file). Check if the file + # is in one of the two sub-trees, and use the relative path to + # check if there is a file-specific optimisation script + for base_path in self._base_paths: + try: + relative_path = fpath.relative_to(base_path) + except ValueError: + # The file is not under the `base_path` - keep on checking + pass + + if relative_path: + # The file was under either source or build. Check if there + # is a file-specific optimisation script: + local_transformation_script = (self.opt_path / + (relative_path.with_suffix('.py'))) + if local_transformation_script.exists(): + return local_transformation_script + return None + + def get_script(self, fpath: Path) -> Path: + """ + This method returns the script to be used for a given filename, or + None if no rule applies (or an explicit exclude rule applies) + + This function will also provided to Fab's PSyclone step, and as such + it will receive the config object, even though it is not used. + + :param fpath: the Fortran source file for which to find a + transformation script. + + :returns: the path to the transformation script, or None if no rule + applies (or an exclude rule applies). + """ + + file_str = str(fpath) + # Search starting from the end, so last rule wins + for rule, file_list in self._rules[::-1]: + for pattern in file_list: + if pattern not in file_str and pattern != "*": + continue + + # Now the pattern matches. Check which rule is used + # (note that file_specific might fall through in case that + # there is no file-specific script) + if rule == PsycloneInfo.FILE_SPECIFIC: + script = self.file_specific_script(fpath) + if script: + return script + if pattern == "*": + # Fall through, i.e. check for other rules + continue + + # Now we have an explicit request for a file-specific + # script, but that script does not exist. + raise FileNotFoundError( + f"Cannot find explicitly requested script '{script}'.") + + if rule == PsycloneInfo.EXCLUDE: + # Exclude pattern matches: + return PsycloneInfo.RESULT_EXCLUDE + + if rule == PsycloneInfo.NO_SCRIPT: + # Exclude pattern matches: + return PsycloneInfo.RESULT_NO_SCRIPT + + opt_script = self.opt_path / rule + if not opt_script.exists(): + raise FileNotFoundError(f"Cannot find script " + f"'{opt_script}' for '{fpath}'.") + return opt_script + + return PsycloneInfo.RESULT_EXCLUDE + + +class PsycloneControl: + """ + This class stores the information from psyclone_control.yaml file(s). + Several files can be read, and latter information will extend the rules + from previous files, and replace the phases executed. + + Details of each phase will be stored in PsycloneInfo instances. + + :param base_paths: + :param fab_base: The FabBase derived application script. This is required + to get site, platform and config information when searching for + PSyclone scripts to be executed. + """ + + def __init__(self, + script_root: Path, + base_paths: list[Path]) -> None: + # Keep a copy in case that the user modifies the list later + self._base_paths = base_paths[:] + self._script_root = script_root + self._all_phases: list[str] = [] + self._psyclone_info: dict[str, PsycloneInfo] = {} + # A list of all PSyclone info files that were read. + # Only used to add useful comments to the yaml output. + self._all_files_read: list[Path] = [] + + @property + def all_phases(self) -> list[str]: + """ + :returns: the list of all PSyclone phases to execute. + """ + return self._all_phases + + def get_info(self, phase: str) -> PsycloneInfo: + """ + Returns the PSyclone information for the specified phase. + + :param phase: the name of the phase. + + :returns: the PSyclone Information for the specified phase. + """ + return self._psyclone_info[phase] + + def to_yaml(self) -> str: + """ + This returns a string representation of the combined read yaml files. + This is useful to be logged to show the actual details used. + + :returns: the string represenation in yaml format of this PSyclone + control instance. + """ + + yaml_dict = {"phases": self._all_phases} + for phase in self._all_phases: + yaml_dict[phase] = self._psyclone_info[phase].get_yaml_dict() + + files_read = '\n'.join(f"# {i}" for i in self._all_files_read) + yaml_string = yaml.dump(yaml_dict, default_flow_style=False, + sort_keys=False) + return f"# Files read:\n{files_read}\n{yaml_string}" + + def read(self, file_path: Path) -> None: + """ + Reads a yaml file, and extends the potentially existing information. + Any phases specified in the new read yaml file will replace the + phases to be executed (i.e. will overwrite what was previously + specified). Any new rule sets will be added as new PsycloneInfo + instance. Rules for an exiting phase will be appended to the + existing information. The precedence handling means that any later + rule will overwrite any previous rule. + + :param file_path: the file_path to read. + """ + + with open(file_path, "r", encoding="utf8") as stream: + dependencies = yaml.safe_load(stream) + self._all_files_read.append(file_path.resolve()) + + # First take phases (if available) + if dependencies.get("phases", None): + self._all_phases = dependencies["phases"] + + for key in dependencies: + if key == "phases": + # Already handled + continue + if key not in self._psyclone_info: + self._psyclone_info[key] = PsycloneInfo(key, + self._base_paths, + self._script_root) + + self._psyclone_info[key].update(dependencies[key]) diff --git a/lfric_build/psyclone_control.yaml b/lfric_build/psyclone_control.yaml new file mode 100644 index 000000000..4fce37377 --- /dev/null +++ b/lfric_build/psyclone_control.yaml @@ -0,0 +1,31 @@ +# This file controls which files are being processed by PSyclone, +# both using PSyclone's transmute ability (transformation generic +# Fortran files), and its DSL (kernel based) capability. + +# First we specify the phases for PSyclone. This allows +# any apps to run transmute and DSL steps in any order just by +# changing (or supplying a different) PSyclone-info file. +# In this file, we only run a single DSL phase. + +phases: + # Just a single phase: running PSyclone in DSL mode + - dsl + +dsl: + comment: "PSylone DSL Phase" + api: lfric + + script_dir: psykal + + # These two directives reproduce the current LFRic default PSyclone triggering: + + # Run optimisation/.../global.py on all x90 files (x90 because this is + # the dsl section, as specified in the artefacts above) + global.py: \* + + # This represents existing functionality: if there is a file-specific .py + # file, use it. The specification of '*' does not trigger an error + # if there is no file_specific script (while explicitly putting a name + # in the file list will trigger an error if the corresponding PSyclone + # script does not exist. This way, errors are caught early). + file_specific: \* diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 57daf31f2..292e3966d 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -13,13 +13,15 @@ import argparse from typing import List -from fab.api import AddFlags, BuildConfig, Category, ToolRepository +from fab.api import BuildConfig, ProfileFlags -from default.setup_script_cray import setup_script_cray -from default.setup_script_gnu import setup_script_gnu -from default.setup_script_intel_classic import setup_script_intel_classic -from default.setup_script_intel_llvm import setup_script_intel_llvm -from default.setup_script_nvidia import setup_script_nvidia +from site_specific.default.setup_script_cray import setup_script_cray +from site_specific.default.setup_script_gnu import setup_script_gnu +from site_specific.default.setup_script_intel_classic import ( + setup_script_intel_classic) +from site_specific.default.setup_script_intel_llvm import ( + setup_script_intel_llvm) +from site_specific.default.setup_script_nvidia import setup_script_nvidia class Config: @@ -58,27 +60,13 @@ def update_toolbox(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' - # First create the default compiler profiles for all available - # compilers. While we have a tool box with exactly one compiler - # in it, compiler wrappers will require more than one compiler - # to be initialised - so we just initialise all of them (including - # the linker): - tr = ToolRepository() - for compiler in (tr[Category.C_COMPILER] + - tr[Category.FORTRAN_COMPILER] + - tr[Category.LINKER]): - # Define a base profile, which contains the common - # compilation flags. This 'base' is not accessible to - # the user, so it's not part of the profile list. Also, - # make it inherit from the default profile '', so that - # a user does not have to specify the 'base' profile. - # Note that we set this even if a compiler is not available. - # This is required in case that compilers are not in PATH, - # so e.g. mpif90-ifort works, but ifort cannot be found. - # We still need to be able to set and query flags for ifort. - compiler.define_profile("base", inherit_from="") - for profile in self.get_valid_profiles(): - compiler.define_profile(profile, inherit_from="base") + # First create the default compiler profiles. + # Define a base profile, which contains the common + # compilation flags. This 'base' is not accessible to + # the user, so it's not part of the profile list. + ProfileFlags.define_profile("base") + for profile in self.get_valid_profiles(): + ProfileFlags.define_profile(profile, inherit_from="base") self.setup_intel_classic(build_config) self.setup_intel_llvm(build_config) @@ -98,6 +86,27 @@ def handle_command_line_options(self, args: argparse.Namespace) -> None: # initialising compilers self._args = args + def update_repos(self, dep_info): + """ + This method is called by the main script to allow each site to + replace the URLs of repos with e.g. local mirrors. + """ + + # A simplified example to use mirrors could be (which would + # typically be implemented in a derived, site-specific class) + # root = Path("/root/of/mirrors") + # mirrors = {"git@github.com:MetOffice/casim.git": root / "casim", + # "git@github.com:MetOffice/jules.git": root / "jules", + # } + # for dependency in dep_info.get_repo_names(): + # repo_infos = dep_info.get_repo_info(dependency) + # for source_ref in repo_infos: + # if source_ref.source in mirrors: + # logger.info(f"Using mirror " + # f"'{mirrors[source_ref.source]}' for " + # f"'{source_ref.source}") + # source_ref.source = mirrors[source_ref.source] + def setup_cray(self, build_config: BuildConfig) -> None: ''' This method sets up the Cray compiler and linker flags. @@ -152,11 +161,3 @@ def setup_nvidia(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' setup_script_nvidia(build_config, self.args) - - def get_path_flags(self, build_config: BuildConfig) -> List[AddFlags]: - ''' - Returns the path-specific flags to be used. - TODO FAB #313: Ideally we have only one kind of flag, but as a quick - work around we provide this method. - ''' - return [] diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index da935bc8b..a6b703aae 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -49,20 +49,14 @@ def setup_script_gnu(build_config: BuildConfig, # The base flags # ============== - # TODO: It should use -Werror=conversion, but: - # Most lfric_atm dependencies contain code with implicit lossy - # conversions. # This should be restricted to only the files/directories # that need it, but this needs Fab updates. gfortran.add_flags( ['-ffree-line-length-none', '-Wall', '-g', - '-Werror=character-truncation', - '-Werror=unused-value', - '-Werror=tabs', + '-Werror=character-truncation', '-Werror=unused-value', + '-Werror=tabs', '-Werror=conversion', '-std=f2008', - '-fdefault-real-8', - '-fdefault-double-8', ], "base") diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 282a322e7..c637d0d4e 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -17,6 +17,7 @@ from fab.api import BuildConfig, Category, Linker, ToolRepository from default.config import Config as DefaultConfig +from nf_config import NfConfig class Config(DefaultConfig): diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 459123655..b23d3bd68 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -26,7 +26,6 @@ from lfric_base import LFRicBase - class MockSiteConfig: """ Creates a mock site config class. @@ -40,6 +39,12 @@ def get_valid_profiles(self) -> List[str]: """ return ["default-profile"] + def update_repos(self, dep_info): + """ + This method is called by the main script to allow each site to + replace the URLs of repos with e.g. local mirrors. + """ + def update_toolbox(self, build_config: BuildConfig) -> None: """ Dummy function where the tool box could be modified @@ -51,12 +56,6 @@ def handle_command_line_options(self, args: argparse.Namespace) -> None: """ self.args = args - def get_path_flags(self, _build_config: BuildConfig) -> List[str]: - """ - :returns: list of path-specific flags. - """ - return [] - @pytest.fixture(name="stub_fortran_compiler", scope='function') def stub_fortran_compiler_init() -> FortranCompiler: @@ -64,8 +63,9 @@ def stub_fortran_compiler_init() -> FortranCompiler: Provides a minimal Fortran compiler. """ compiler = FortranCompiler('some Fortran compiler', 'sfc', 'stub', - r'([\d.]+)', openmp_flag='-omp', - module_folder_flag='-mods') + r'([\d.]+)') + compiler["openmp"] = '-omp' + compiler["module-out-folder"] = '-mods' return compiler @@ -75,7 +75,8 @@ def stub_c_compiler_init() -> CCompiler: Provides a minimal C compiler. """ compiler = CCompiler("some C compiler", "scc", "stub", - version_regex=r"([\d.]+)", openmp_flag='-omp') + version_regex=r"([\d.]+)") + compiler["openmp"] = '-omp' return compiler @@ -226,6 +227,8 @@ def create_frame_info(filename): ] monkeypatch.setattr('inspect.stack', lambda: mock_stack) monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + psyclone_control = mock_base_dir / "psyclone_control.yaml" + psyclone_control.write_text("phases:", encoding='utf-8') lfric_base = LFRicBase(name="test", app_dir=tmp_path / "app_dir") @@ -234,6 +237,20 @@ def create_frame_info(filename): assert lfric_base.app_dir == tmp_path / "app_dir" +def test_python_search_path(monkeypatch): + ''' + Test that new paths can be added to the Python search path. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + lfric_base = LFRicBase(name="test_name", app_dir=Path(".")) + tools_path = str(lfric_base.lfric_core_root / "infrastructure" / "build" / + "psyclone") + # pylint: disable=protected-access + assert lfric_base._add_python_paths == [tools_path] + lfric_base.add_python_search_path("/special_path") + assert lfric_base._add_python_paths == [tools_path, "/special_path"] + + def test_require_openmp(monkeypatch, caplog) -> None: ''' Tests that using `-no-openmp` will abort with correct @@ -350,7 +367,6 @@ def test_setup_site_specific_location(monkeypatch) -> None: # Check paths added correctly base_dir = Path(inspect.getfile(LFRicBase)).parent assert str(base_dir) in sys.path - assert str(base_dir / "site_specific") in sys.path # Restore path sys.path = old_path @@ -409,7 +425,11 @@ def test_grab_files_step(monkeypatch) -> None: # PSyclone config directory mock.call(lfric_base.config, src=mock_core/'etc', - dst_label='psyclone_config') + dst_label='psyclone_config'), + # PSyclone optimisation scripts + mock.call(lfric_base.config, + src=Path('optimisation'), + dst_label='optimisation') ] # Check both number of calls and call arguments @@ -560,91 +580,106 @@ def test_get_rose_meta(monkeypatch) -> None: lfric_base = LFRicBase(name="test", app_dir=Path(".")) assert lfric_base.get_rose_meta() is None +class TestFullSetup(): -def test_analyse_step(monkeypatch) -> None: - '''Tests analysis step configuration and execution''' + def setup(self, monkeypatch, argv: Optional[list[str]] = None) -> None: + """ + """ + self.mpatch = monkeypatch + if not argv: + argv = ["lfric_base.py"] + self.mpatch.setattr(sys, "argv", argv) - # Test case 1: No ignore_dependencies argument specified - monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + # Create and setup mocks + self.mock_analyse = mock.MagicMock() + self.mpatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', + self.mock_analyse) - # Create mocks - mock_analyse = mock.MagicMock() - mock_preprocess = mock.MagicMock() - mock_psyclone = mock.MagicMock() + self.mock_preprocess = mock.MagicMock() + self.mpatch.setattr('lfric_base.preprocess_x90', self.mock_preprocess) - # Setup mocks - monkeypatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', - mock_analyse) + self.mock_psyclone_step = mock.MagicMock() + self.mpatch.setattr('lfric_base.LFRicBase.psyclone_step', + self.mock_psyclone_step) - lfric_base = LFRicBase(name="test", app_dir=Path(".")) + # Set up monkeypatch for module level import + self.mock_psyclone = mock.MagicMock() + monkeypatch.setattr('lfric_base.psyclone', self.mock_psyclone) - # Mock instance methods - monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) - monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) - - # The PSyclone step will modify sys.path (to allow import of - # psyclone_tools by PSyclone scripts). Make sure sys.path is unchanged: - old_sys_path = sys.path[:] - # Call analyse_step (which calls PSyclone) - lfric_base.analyse_step() - assert sys.path == old_sys_path - - # Verify method calls - mock_preprocess.assert_called_once() - mock_psyclone.assert_called_once() - - # Verify analyse called with correct default ignore_dependencies - expected_ignore = ['netcdf', 'mpi', 'mpi_f08', 'yaxt', - 'xios', 'icontext', 'mod_wait'] - mock_analyse.assert_called_once_with( - ignore_dependencies=expected_ignore, - find_programs=False - ) + self.lfric_base = LFRicBase(name="test", app_dir=Path(".")) - # Test case 2: Custom ignore_dependencies arguments specified - custom_ignore = ['custom_dep1', 'custom_dep2'] - mock_analyse.reset_mock() - mock_preprocess.reset_mock() - mock_psyclone.reset_mock() + self.mock_psyclone_config = "/mock/psyclone.cfg" + # Patch instance methods. Return a copy to avoid that + # PSyclone modified these lists in the lambdas when it modifies the list + monkeypatch.setattr(self.lfric_base, 'get_psyclone_config', + lambda: self.mock_psyclone_config) - lfric_base = LFRicBase(name="test", app_dir=Path(".")) - monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) - monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) - - # Call analyse_step - lfric_base.analyse_step(ignore_dependencies=custom_ignore) - - # Verify methods still called - mock_preprocess.assert_called_once() - mock_psyclone.assert_called_once() - - # Verify analyse called with custom_ignore added to ignore list - expected_ignore = ['custom_dep1', 'custom_dep2', 'netcdf', 'mpi', - 'mpi_f08', 'yaxt', 'xios', 'icontext', - 'mod_wait'] - mock_analyse.assert_called_once_with( - ignore_dependencies=expected_ignore, - find_programs=False - ) + def test_analyse_no_ignore(self, monkeypatch) -> None: + """ + Tests analysis step configuration and execution, + if not additional dependencies are specified. + """ -def test_preprocess_x90_step(monkeypatch) -> None: - ''' - Tests preprocessing of X90 files. - ''' - monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + self.setup(monkeypatch) + + # The PSyclone step will modify sys.path (to allow import of + # psyclone_tools by PSyclone scripts). Make sure sys.path is unchanged: + old_sys_path = sys.path[:] + # Call analyse_step (which calls PSyclone) + self.lfric_base.analyse_step() + assert sys.path == old_sys_path + + # Verify method calls + self.mock_preprocess.assert_called_once() + self.mock_psyclone_step.assert_called_once() + + # Verify analyse called with correct default ignore_dependencies + expected_ignore = ['netcdf', 'mpi', 'mpi_f08', 'yaxt', + 'xios', 'icontext', 'mod_wait'] + self.mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) - mock_preproc = mock.MagicMock() - monkeypatch.setattr('lfric_base.preprocess_x90', mock_preproc) + def test_analyse_ignore_dependency(self, monkeypatch) -> None: + """ + Tests analysis step configuration and execution when + additional dependencies are specified. + """ - lfric_base = LFRicBase(name="test", app_dir=Path(".")) - lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) - lfric_base.preprocess_x90_step() + self.setup(monkeypatch) - mock_preproc.assert_called_once_with( - lfric_base.config, - common_flags=["-flag1", "-flag2"] - ) + # Call analyse_step (which calls PSyclone) + custom_ignore = ['custom_dep1', 'custom_dep2'] + self.lfric_base.analyse_step(ignore_dependencies=custom_ignore) + + # Verify method calls + self.mock_preprocess.assert_called_once() + self.mock_psyclone_step.assert_called_once() + + # Verify analyse called with correct default ignore_dependencies + expected_ignore = ['custom_dep1', 'custom_dep2', 'netcdf', 'mpi', + 'mpi_f08', 'yaxt', 'xios', 'icontext', + 'mod_wait'] + self.mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) + + def test_preprocess_x90_step(self, monkeypatch) -> None: + ''' + Tests preprocessing of X90 files. + ''' + self.setup(monkeypatch) + + self.lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) + self.lfric_base.preprocess_x90_step() + + self.mock_preprocess.assert_called_once_with( + self.lfric_base.config, + common_flags=["-flag1", "-flag2"] + ) def test_psyclone_step(monkeypatch) -> None: @@ -691,7 +726,7 @@ def test_get_psyclone_config(monkeypatch) -> None: config_args = lfric_base.get_psyclone_config() assert config_args == str(lfric_base.config.source_root / - 'psyclone_config/psyclone.cfg') + 'psyclone_config' / 'psyclone.cfg') def test_get_transformation_script(monkeypatch, tmp_path) -> None: @@ -703,47 +738,70 @@ def test_get_transformation_script(monkeypatch, tmp_path) -> None: # Create LFRicBase instance with mocked site/platform lfric_base = LFRicBase(name="test", app_dir=Path(".")) - # Create mock config + # Create mock config, and insert it into the lfric base instance + # (to mock the paths used here) config = mock.MagicMock() config.source_root = tmp_path - config.build_output = tmp_path / "build" + config._build_output = tmp_path / "build" config.build_output.mkdir() + lfric_base._config = config + + optimisation_folder_path = (tmp_path / "optimisation" / "default-default" / + "psykal") + + # Set a current psyclone phase in the object, so the correct rule + # (for dsl) is picked: + psy_info = lfric_base._psyclone_control.get_info("dsl") + psy_info._opt_path = optimisation_folder_path + # Set tmp_path as the base_paths (which is used to detect + # the relative path when checking for file-specific scripts). + psy_info._base_paths = [tmp_path] + lfric_base._current_psyclone_info = psy_info # Create x90 test source file - source_path = tmp_path / "some/path" + source_path = tmp_path / "some" / "path" source_path.mkdir(parents=True) + + # Test case 1: No optimisation directory, no transformation script test_file = source_path / "file.x90" test_file.touch() + with pytest.raises(FileNotFoundError) as err: + lfric_base.get_transformation_script(test_file, config) - # Test case 1: x90 file not in source or build directories - outside_file = tmp_path.parent / "outside.x90" - assert lfric_base.get_transformation_script(outside_file, config) is None - - # Test case 2: No optimisation directory, no transformation script - assert lfric_base.get_transformation_script(test_file, config) is None - - # Test case 3: No PSykal but optimisation directory - optimisation_folder_path = (tmp_path / "optimisation" / "default-default" / - "psykal") + # Test case 2: No PSykal but optimisation directory global_script = optimisation_folder_path / "global.py" global_script.parent.mkdir(parents=True) global_script.touch() # No file-specific transformation script, use global script - other_file = tmp_path / "other/path/test.x90" + other_file = tmp_path / "other" / "path" / "test.x90" other_file.parent.mkdir(parents=True) other_file.touch() assert (lfric_base.get_transformation_script(other_file, config) == global_script) - # Test case 4: Psykal directory exists - psykal_path = tmp_path / "optimisation/default-default/psykal" + # Test case 3: Psykal directory exists + psykal_path = tmp_path / "optimisation" / "default-default" / "psykal" # Create specific transformation script in psykal dir - specific_script = psykal_path / "some/path/file.py" + specific_script = psykal_path / "some" / "path" / "file.py" specific_script.parent.mkdir(parents=True) specific_script.touch() # Use specific script in psykal directory assert lfric_base.get_transformation_script(test_file, config) == \ specific_script + + # Test case 4: Return exclude (which should not happen, the function + # should only be called for files that have a script) + psy_info._rules = [] + with pytest.raises(ValueError) as err: + lfric_base.get_transformation_script(test_file, config) + assert ("PSyclone transformation script returned " + "'PsycloneInfo.RESULT_EXCLUDE', which should not happen." + in str(err)) + + # Test case 5: Return None if the PSyclone control file explicitly + # requests to run PSyclone without a script. + psy_info._rules = [(psy_info.NO_SCRIPT, "*")] + assert lfric_base.get_transformation_script(test_file, config) is None diff --git a/lfric_build/tests/nf_config_test.py b/lfric_build/tests/nf_config_test.py index 464796f8a..7d6f6a1b9 100644 --- a/lfric_build/tests/nf_config_test.py +++ b/lfric_build/tests/nf_config_test.py @@ -33,7 +33,6 @@ def test_constructor() -> None: assert nfc.category == Category.MISC assert nfc.name == "nf-config" assert nfc.exec_name == "nf-config" - assert nfc.get_flags() == [] def test_nf_config_check_available(fake_process: FakeProcess) -> None: diff --git a/lfric_build/tests/psyclone_control_test.py b/lfric_build/tests/psyclone_control_test.py new file mode 100644 index 000000000..ca68960cc --- /dev/null +++ b/lfric_build/tests/psyclone_control_test.py @@ -0,0 +1,207 @@ +# ############################################################################ +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +# ############################################################################ + +""" +Unit tests for the psyclone_control module. +""" + +from pathlib import Path +import pytest +import yaml + +from psyclone_control import PsycloneInfo, PsycloneControl + + +def test_psyclone_info_properties(): + """ + Test initial properties and getters of PsycloneInfo. + """ + base_paths = [Path("/base1"), Path("/base2")] + script_root = Path("/scripts") + + info = PsycloneInfo( + name="test_phase", base_paths=base_paths, script_root=script_root + ) + + assert info.name == "test_phase" + assert info.comment == "" + assert info.api == "" + assert info.opt_path == Path() + + +@pytest.mark.parametrize("escaped_wildcard", ["\\*", "'*'", '"*"']) +def test_psyclone_info_wildcard_handling(escaped_wildcard): + """ + Verify different yaml wildcard string variants parse as expected. + """ + info = PsycloneInfo("phase", [], Path()) + # Indirectly hit _read_rule via update + yaml_dict = {"some_script.py": escaped_wildcard} + info.update(yaml_dict) + + assert info._rules == [("some_script.py", ["*"])] + + +def test_psyclone_control_read_and_to_yaml(tmp_path): + """ + Test full workflow: reading configuration files and generating YAML output. + """ + yaml_content_1 = """ +phases: + - dsl +dsl: + comment: "PSyclone DSL Phase" + api: lfric + script_dir: psykal + global.py: \\* + file_specific: \\* +""" + yaml_file_1 = tmp_path / "psyclone_info.yaml" + yaml_file_1.write_text(yaml_content_1, encoding="utf-8") + + # Second YAML payload tests appending/overwriting phase specifications + yaml_content_2 = """ +phases: + - dsl + - secondary +secondary: + comment: "Secondary phase" + script_dir: alternative +""" + yaml_file_2 = tmp_path / "psyclone_info_override.yaml" + yaml_file_2.write_text(yaml_content_2, encoding="utf-8") + + base_paths = [tmp_path / "src", tmp_path / "build"] + script_root = tmp_path / "scripts" + + pc = PsycloneControl(script_root=script_root, base_paths=base_paths) + + # Read first file + pc.read(yaml_file_1) + assert pc.all_phases == ["dsl"] + + info_dsl = pc.get_info("dsl") + assert info_dsl.comment == "PSyclone DSL Phase" + assert info_dsl.api == "lfric" + assert info_dsl.opt_path == script_root / "psykal" + + # Read second file to test incremental overrides + pc.read(yaml_file_2) + assert pc.all_phases == ["dsl", "secondary"] + assert pc.get_info("secondary").comment == "Secondary phase" + + # Test YAML text generation output matches structures + yaml_out = pc.to_yaml() + assert f"# {yaml_file_1.resolve()}" in yaml_out + assert f"# {yaml_file_2.resolve()}" in yaml_out + + parsed_out = yaml.safe_load(yaml_out) + assert parsed_out["phases"] == ["dsl", "secondary"] + assert parsed_out["dsl"]["api"] == "lfric" + + +def test_file_specific_script_resolution(tmp_path): + """ + Validate looking up file-specific scripts inside source tree trees. + """ + base_src = tmp_path / "src" + base_build = tmp_path / "build" + script_root = tmp_path / "scripts" + + # Define an active script path destination directory + opt_dir = script_root / "psykal" + opt_dir.mkdir(parents=True) + + info = PsycloneInfo( + name="dsl", base_paths=[base_src, base_build], script_root=script_root + ) + info.update({"script_dir": "psykal"}) + + # Case 1: Target file is out of any base path boundary + external_file = tmp_path / "outside" / "some_mod.x90" + assert info.file_specific_script(external_file) is None + + # Case 2: Target inside src directory, but no companion exists + src_file = base_src / "kernel" / "some_mod.x90" + assert info.file_specific_script(src_file) is None + + # Case 3: Script target exists matching the relative structure + expected_script = opt_dir / "kernel" / "some_mod.py" + expected_script.parent.mkdir(parents=True, exist_ok=True) + expected_script.touch() + + assert info.file_specific_script(src_file) == expected_script + + +def test_get_script_matching_logic(tmp_path): + """ + Verify filtering behavior, fallback hierarchies, and exception conditions. + """ + base_src = tmp_path / "src" + script_root = tmp_path / "scripts" + opt_dir = script_root / "psykal" + opt_dir.mkdir(parents=True) + + info = PsycloneInfo( + name="dsl", base_paths=[base_src], script_root=script_root + ) + + # Setup rules: non-matching pattern, NO_SCRIPT rule, then explicit scripts + yaml_config = { + "script_dir": "psykal", + "exclude": "ignored_module.x90", + "no_script": "skipped_module.x90", + "missing_script.py": "broken_module.x90", + "valid_script.py": "good_module.x90", + "file_specific": "\\*", + } + info.update(yaml_config) + + # 1. Test standard pattern mismatch fallback + assert ( + info.get_script(Path("unmatched_file.x90")) + == PsycloneInfo.RESULT_EXCLUDE + ) + + # 2. Test explicit EXCLUDE rules matching + assert ( + info.get_script(Path("ignored_module.x90")) + == PsycloneInfo.RESULT_EXCLUDE + ) + + # 3. Test explicit NO_SCRIPT matching + assert ( + info.get_script(Path("skipped_module.x90")) + == PsycloneInfo.RESULT_NO_SCRIPT + ) + + # 4. Test explicit rule script missing physically on storage disk + with pytest.raises( + FileNotFoundError, match="Cannot find script '.*missing_script.py'" + ): + info.get_script(Path("broken_module.x90")) + + # 5. Test explicit rule script that exists successfully + valid_script_path = opt_dir / "valid_script.py" + valid_script_path.touch() + assert info.get_script(Path("good_module.x90")) == valid_script_path + + # 6. Test file_specific script with non-existent explicit definition + info_strict = PsycloneInfo( + name="dsl", base_paths=[base_src], script_root=script_root + ) + info_strict.update( + {"script_dir": "psykal", "file_specific": "explicit_custom.x90"} + ) + with pytest.raises( + FileNotFoundError, match="Cannot find explicitly requested script" + ): + info_strict.get_script(base_src / "explicit_custom.x90") + + # 7. File_specific if the script exists: + valid_script_path = opt_dir / "file_specific.py" + valid_script_path.touch() + assert info.get_script(base_src / "file_specific.x90") == valid_script_path