From 4dbac71448248190e467d6b9b08737fb9711f3ad Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 23 Jan 2026 17:40:30 +1100 Subject: [PATCH 01/95] #240 Added new base class for Fab scripts. --- infrastructure/build/fab/lfric_base.py | 437 +++++++++++++++++++++++++ 1 file changed, 437 insertions(+) create mode 100755 infrastructure/build/fab/lfric_base.py diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py new file mode 100755 index 000000000..2ecce6af6 --- /dev/null +++ b/infrastructure/build/fab/lfric_base.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +# ############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +# ############################################################################## + +''' +This is an OO basic interface to FAB. It allows the typical LFRic +applications to only modify very few settings to have a working FAB build +script. +''' + +import argparse +import os +from pathlib import Path +import sys +from typing import List, Optional, Iterable, Union + +from fab.api import (ArtefactSet, BuildConfig, Exclude, grab_folder, Include, + input_to_output_fpath, preprocess_x90, psyclone, + SuffixFilter) +from fab.fab_base.fab_base import FabBase + +from configurator import configurator +from rose_picker_tool import get_rose_picker +from templaterator import Templaterator +from transmute_step import TransmuteStep + + +class LFRicBase(FabBase): + ''' + This is the base class for all LFRic FAB scripts. + + :param name: the name to be used for the workspace. Note that + the name of the compiler will be added to it. + :param root_symbol: the symbol (or list of symbols) of the main + programs. Defaults to the parameter `name` if not specified. + + ''' + # pylint: disable=too-many-instance-attributes + def __init__(self, name: str, + root_symbol: Optional[Union[List[str], str]] = None + ): + + # List of all precision preprocessor symbols and their default. + # Used to add corresponding command line options, and then to define + # the preprocessor definitions. + self._all_precisions = [("RDEF_PRECISION", "64"), + ("R_SOLVER_PRECISION", "32"), + ("R_TRAN_PRECISION", "64"), + ("R_BL_PRECISION", "64")] + + super().__init__(name) + + this_file = Path(__file__) + # The root directory of the LFRic Core + self._lfric_core_root = this_file.parents[3] + + # If the user wants to overwrite the default root symbol (which + # is `name`): + if root_symbol: + self.set_root_symbol(root_symbol) + + self._psyclone_config = (self.config.source_root / 'psyclone_config' / + 'psyclone.cfg') + + def define_command_line_options( + self, + parser: Optional[argparse.ArgumentParser] = None + ) -> argparse.ArgumentParser: + ''' + This adds LFRic specific command line options to the base class + define_command_line_option. Currently, --rose_picker and + precision-related options are added. + + :param parser: optional a pre-defined argument parser. + + :returns: the argument parser with the LFRic specific options added. + ''' + parser = super().define_command_line_options() + + parser.add_argument( + '--rose_picker', '-rp', type=str, default="system", + help="Version of rose_picker. Use 'system' to use an installed " + "version.") + + parser.add_argument( + '--no-xios', action="store_true", default=False, + help="Disable compilation with XIOS.") + + parser.add_argument('--transmute', action="append", + help="Specify a transmute file which will trigger" + "additional PSyclone processing.") + + # Precision related command line arguments + # ---------------------------------------- + group = parser.add_argument_group( + title="Precisions", + description="Arguments related to setting the floating " + "point precision.") + + group.add_argument( + '--precision-default', type=str, default=None, + choices=['32', '64'], help="Default precision for reals.") + + # We need to distinguish if a user specified a value (even if it is + # the default), or not. Use the following action for argparse: + class StoreWithFlag(argparse.Action): + """ + Helper class to add a `XX_specified` entry for command line + options that the user has explicitly specified. + """ + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + setattr(namespace, f"{self.dest}_specified", True) + + for prec_name, default in self._all_precisions: + lower_name = prec_name.lower() + group.add_argument( + f'--{lower_name}', type=str, choices=['32', '64'], + default=default, action=StoreWithFlag, + help=f"Precision for '{prec_name}'. Default will be " + f"overwritten by ${prec_name} or --precision-default " + f"in this order.") + + return parser + + @property + def lfric_core_root(self) -> Path: + ''' + :returns: the root directory of the LFRic core repository. + ''' + return self._lfric_core_root + + def setup_site_specific_location(self): + ''' + This method adds the required directories for site-specific + configurations to the Python search path. We want to add the + directory where this lfric_base class is located, and not the + directory in which the application script is (which is what + baf base would set up). + ''' + this_dir = Path(__file__).parent + sys.path.insert(0, str(this_dir)) + # We need to add the 'site_specific' directory to the path, so + # each config can import from 'default' (instead of having to + # use 'site_specific.default', which would hard-code the name + # `site_specific` in more scripts). + sys.path.insert(0, str(this_dir / "site_specific")) + + def define_preprocessor_flags_step(self) -> None: + ''' + This method overwrites the base class define_preprocessor_flags. + It uses add_preprocessor_flags to set up preprocessing flags for LFRic + applications. This includes: + - various floating point precision related directives + - Use of XIOS (if not disabled using --no-xios command line option) + - Disabling MPI (if disabled using --no-mpi) + ''' + preprocessor_flags: List[str] = [] + + # Take the value of --precision-default (or None if not specified): + generic_default = self.args.precision_default + + # Check all required precision defines + for prec_name, prec_default in self._all_precisions: + # Check if a value was specified on the command line: + if getattr(self.args, f"{prec_name.lower()}_specified", False): + value = getattr(self.args, prec_name.lower()) + preprocessor_flags.append(f"-D{prec_name}={value}") + continue + # Check for environment variable which can overwrite the default: + env_precision = os.environ.get(prec_name) + if env_precision: + preprocessor_flags.append(f"-D{prec_name}={env_precision}") + continue + + # No command line option for the current precision name. + # Check if a default was set (--precision-default) + if generic_default: + preprocessor_flags.append(f"-D{prec_name}=" + f"{generic_default}") + else: + # Otherwise, use the default for this precision + preprocessor_flags.append(f"-D{prec_name}={prec_default}") + + # core/components/lfric-xios/build/import.mk + if not self.args.no_xios: + preprocessor_flags.append('-DUSE_XIOS') + + if not self.config.mpi: + preprocessor_flags.append("-DNO_MPI") + + self.add_preprocessor_flags(preprocessor_flags) + + def get_linker_flags(self) -> List[str]: + ''' + This method overwrites the base class get_liner_flags. It passes the + libraries that LFRic uses to the linker. Currently, these libraries + include yaxt, xios, netcdf and hdf5. + + :returns: list of flags for the linker. + ''' + libs = ['yaxt', 'xios', 'netcdf', 'hdf5'] + return libs + super().get_linker_flags() + + def grab_files_step(self) -> None: + ''' + This method overwrites the base class grab_files_step. It includes all + the LFRic core directories that are commonly required for building + LFRic applications. It also grabs the psydata directory for profiling, + if required. + ''' + dirs = ['infrastructure/source/', + 'components/driver/source/', + 'components/inventory/source/', + 'components/science/source/', + 'components/lfric-xios/source/', + ] + + # pylint: disable=redefined-builtin + for dir in dirs: + grab_folder(self.config, src=self.lfric_core_root / dir, + dst_label='') + + # Copy the PSyclone Config file into a separate directory + dir = "etc" + grab_folder(self.config, src=self.lfric_core_root / dir, + dst_label='psyclone_config') + + def find_source_files_step( + self, + path_filters: Optional[Iterable[Union[Exclude, Include]]] = None + ) -> None: + ''' + This method overwrites the base class find_source_files_step. + It first calls the configurator_step to set up the configurator. + Then it finds all the source files in the LFRic core directories, + excluding the unit tests. Finally, it calls the templaterator_step. + + :param path_filters: optional list of path filters to be passed to + Fab find_source_files, default is None. + :type path_filters: Optional[Iterable[Exclude, Include]] + ''' + self.configurator_step() + + path_filter_list = list(path_filters) if path_filters else [] + path_filter_list.append(Exclude('unit-test', '/test/')) + super().find_source_files_step(path_filters=path_filter_list) + + self.templaterator_step(self.config) + + def configurator_step( + self, + include_paths: Optional[list[Path]] = None) -> None: + ''' + This method first gets the rose meta data information by calling + get_rose_meta. If the rose meta data is available, it then get the + rose picker tool by calling the get_rose_picker. Finally, it runs + the LFRic configurator with the LFRic core and apps sources by calling + configurator. + + :param include_paths: optional additional include paths + ''' + rose_meta = self.get_rose_meta() + if rose_meta: + # Get the right version of rose-picker, depending on + # command line option (defaulting to v2.0.0) + # TODO: Ideally we would just put this into the toolbox, + # but atm we can't put several tools of one category in + # (so ToolBox will need to support more than one MISC tool) + rp = get_rose_picker(self.args.rose_picker) + # Ideally we would want to get all source files created in + # the build directory, but then we need to know the list of + # files to add them to the list of files to process. Instead, + # we create the files in the source directory, and find them + # there later. + include_paths = include_paths or [] + configurator(self.config, lfric_core_source=self.lfric_core_root, + rose_meta_conf=rose_meta, + include_paths=include_paths, + rose_picker=rp) + + def templaterator_step(self, config: BuildConfig) -> None: + ''' + This method runs the LFRic templaterator Fab tool. + + :param config: the Fab build configuration + :type config: :py:class:`fab.BuildConfig` + ''' + base_dir = self.lfric_core_root / "infrastructure" / "build" / "tools" + + templaterator = Templaterator(base_dir/"Templaterator") + config.artefact_store["template_files"] = set() + t90_filter = SuffixFilter(ArtefactSet.INITIAL_SOURCE_FILES, + [".t90", ".T90"]) + template_files = t90_filter(config.artefact_store) + # Don't bother with parallelising this, atm there is only one file: + for template_file in template_files: + out_dir = input_to_output_fpath(config=config, + input_path=template_file).parent + out_dir.mkdir(parents=True, exist_ok=True) + templ_r32 = {"kind": "real32", "type": "real"} + templ_r64 = {"kind": "real64", "type": "real"} + templ_i32 = {"kind": "int32", "type": "integer"} + for key_values in [templ_r32, templ_r64, templ_i32]: + out_file = out_dir / f"field_{key_values['kind']}_mod.f90" + templaterator.process(template_file, out_file, + key_values=key_values) + config.artefact_store.add(ArtefactSet.FORTRAN_COMPILER_FILES, + out_file) + + def get_rose_meta(self) -> Union[Path, None]: + ''' + This method returns the path to the rose meta data config file. + Currently, it returns none. It's up to the LFRic applications to + overwrite if required. + ''' + return None + + def analyse_step( + self, + ignore_dependencies: Optional[Iterable[str]] = None, + find_programs: bool = False + ) -> None: + ''' + The method overwrites the base class analyse_step. + For LFRic, it first runs the preprocess_x90_step and then runs + psyclone_step. Finally, it calls Fab's analyse for dependency + analysis, ignoring the third party modules that are commonly + used by LFRic. + ''' + if ignore_dependencies is None: + ignore_dependencies = [] + # core/infrastructure/build/import.mk + ignore_dep_list = list(ignore_dependencies) + ignore_dep_list += ['netcdf', 'mpi', 'mpi_f08', 'yaxt'] + # From core/components/lfric-xios/build/import.mk + ignore_dep_list += ['xios', 'icontext', 'mod_wait'] + + self.preprocess_x90_step() + + # Do the transmute step - contained in a separate object + # to keep the file size smaller. + transmute = TransmuteStep(self.config, self.site, self.platform) + transmute.transmute_step(self.args.transmute) + + self.psyclone_step(ignore_dependencies=ignore_dep_list) + super().analyse_step( + ignore_dependencies=ignore_dep_list, + find_programs=find_programs) + + def preprocess_x90_step(self) -> None: + """ + Invokes the Fab preprocess step for all X90 files. + """ + # TODO: Fab does not support path-specific flags for X90 files. + preprocess_x90(self.config, + common_flags=self.preprocess_flags_common) + + def psyclone_step( + self, + ignore_dependencies: Optional[Iterable[str]] = None, + additional_parameters: Optional[list[str]] = None + ) -> None: + ''' + This method runs Fab's psyclone. It first sets the additional psyclone + command line arguments by calling get_psyclone_config to get the + PSyclone configuration file and by calling + `get_additional_psyclone_options` to get additional psyclone command + line set by the user, e.g. for profiling, if any. Finally, Fab's + psyclone is called with the Fab build configuration, the kernel root + directory, the transformation script got through calling + `get_transformation_script`, the api, and the additional psyclone + command line arguments. + + :param ignore_dependencies: + :param additional_parameters: optional additional parameter for the + PSyclone. + ''' + psyclone_cli_args = self.get_psyclone_config() + psyclone_cli_args.extend(self.get_additional_psyclone_options()) + if additional_parameters: + psyclone_cli_args.extend(additional_parameters) + + psyclone(self.config, kernel_roots=[(self.config.build_output / + "kernel")], + transformation_script=self.get_transformation_script, + api="dynamo0.3", + cli_args=psyclone_cli_args, + ignore_dependencies=ignore_dependencies) + + def get_psyclone_config(self) -> List[str]: + ''' + :returns: the command line options to pick the right + PSyclone config file. + ''' + return ["--config", str(self._psyclone_config)] + + def get_additional_psyclone_options(self) -> List[str]: + ''' + A placeholder for additional PSyclone comand line options. + ''' + return [] + + def get_transformation_script(self, fpath: Path, + config: BuildConfig) -> Optional[Path]: + ''' + This method returns the path to the transformation script that PSyclone + will use for each x90 file. It first checks if there is a specific + transformation script for the x90 file. If not, it will see whether a + global transformation script can be used. + + :param fpath: the path to the file being processed. + :param config: the FAB BuildConfig instance. + :returns: the transformation script to be used by PSyclone. + ''' + # Newer LFRic versions have a psykal directory + optimisation_path = (config.source_root / "optimisation" / + f"{self.site}-{self.platform}" / "psykal") + relative_path = None + for base_path in [config.source_root, config.build_output]: + try: + relative_path = fpath.relative_to(base_path) + except ValueError: + pass + if relative_path: + local_transformation_script = (optimisation_path / + (relative_path.with_suffix('.py'))) + if local_transformation_script.exists(): + return local_transformation_script + + global_transformation_script = optimisation_path / 'global.py' + if global_transformation_script.exists(): + return global_transformation_script + return None From 2254b3122c9f6cf0b52d6aed6ee735c6efbc163d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 23 Jan 2026 18:18:18 +1100 Subject: [PATCH 02/95] #240 Removed support for transmute step to keep this PR smaller. --- infrastructure/build/fab/lfric_base.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 2ecce6af6..fef3876ae 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -25,7 +25,6 @@ from configurator import configurator from rose_picker_tool import get_rose_picker from templaterator import Templaterator -from transmute_step import TransmuteStep class LFRicBase(FabBase): @@ -89,10 +88,6 @@ def define_command_line_options( '--no-xios', action="store_true", default=False, help="Disable compilation with XIOS.") - parser.add_argument('--transmute', action="append", - help="Specify a transmute file which will trigger" - "additional PSyclone processing.") - # Precision related command line arguments # ---------------------------------------- group = parser.add_argument_group( @@ -340,12 +335,6 @@ def analyse_step( ignore_dep_list += ['xios', 'icontext', 'mod_wait'] self.preprocess_x90_step() - - # Do the transmute step - contained in a separate object - # to keep the file size smaller. - transmute = TransmuteStep(self.config, self.site, self.platform) - transmute.transmute_step(self.args.transmute) - self.psyclone_step(ignore_dependencies=ignore_dep_list) super().analyse_step( ignore_dependencies=ignore_dep_list, From c1e87d1e9e71c5c84fc025c692ad4c7e4ab702fa Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 23 Jan 2026 18:24:03 +1100 Subject: [PATCH 03/95] #240 Add all skeleton fab script, helper classes and site-configuration to compile skeleton. --- applications/skeleton/fab_skeleton.py | 70 ++++++++ infrastructure/build/fab/configurator.py | 90 ++++++++++ infrastructure/build/fab/rose_picker_tool.py | 113 ++++++++++++ .../fab/site_specific/default/__init__.py | 0 .../build/fab/site_specific/default/config.py | 163 ++++++++++++++++++ .../fab/site_specific/default/setup_cray.py | 119 +++++++++++++ .../fab/site_specific/default/setup_gnu.py | 113 ++++++++++++ .../default/setup_intel_classic.py | 110 ++++++++++++ .../site_specific/default/setup_intel_llvm.py | 92 ++++++++++ .../fab/site_specific/default/setup_nvidia.py | 110 ++++++++++++ .../fab/site_specific/meto_ex1a/config.py | 42 +++++ .../build/fab/site_specific/ncas_ex/config.py | 52 ++++++ .../fab/site_specific/nci_gadi/__init__.py | 0 .../fab/site_specific/nci_gadi/config.py | 114 ++++++++++++ .../fab/site_specific/niwa_xc50/config.py | 61 +++++++ infrastructure/build/fab/templaterator.py | 58 +++++++ 16 files changed, 1307 insertions(+) create mode 100755 applications/skeleton/fab_skeleton.py create mode 100755 infrastructure/build/fab/configurator.py create mode 100755 infrastructure/build/fab/rose_picker_tool.py create mode 100644 infrastructure/build/fab/site_specific/default/__init__.py create mode 100644 infrastructure/build/fab/site_specific/default/config.py create mode 100644 infrastructure/build/fab/site_specific/default/setup_cray.py create mode 100644 infrastructure/build/fab/site_specific/default/setup_gnu.py create mode 100644 infrastructure/build/fab/site_specific/default/setup_intel_classic.py create mode 100644 infrastructure/build/fab/site_specific/default/setup_intel_llvm.py create mode 100644 infrastructure/build/fab/site_specific/default/setup_nvidia.py create mode 100644 infrastructure/build/fab/site_specific/meto_ex1a/config.py create mode 100644 infrastructure/build/fab/site_specific/ncas_ex/config.py create mode 100644 infrastructure/build/fab/site_specific/nci_gadi/__init__.py create mode 100644 infrastructure/build/fab/site_specific/nci_gadi/config.py create mode 100644 infrastructure/build/fab/site_specific/niwa_xc50/config.py create mode 100755 infrastructure/build/fab/templaterator.py diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py new file mode 100755 index 000000000..8bd4d6e1e --- /dev/null +++ b/applications/skeleton/fab_skeleton.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# ############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +# ############################################################################## + +'''A FAB build script for applications/skeleton. It relies on +the LFRicBase class contained in the infrastructure directory. +''' + +import logging +from pathlib import Path +import sys + +from fab.steps.grab.folder import grab_folder + +# We need to import the base class: +sys.path.insert(0, str(Path(__file__).parents[2] / "infrastructure" / + "build" / "fab")) + +from lfric_base import LFRicBase # noqa: E402 + + +class FabSkeleton(LFRicBase): + """ + A Fab-based build script for skeleton. It relies on the LFRicBase class + to implement the actual functionality, and only provides the required + source files. + + :param name: The name of the application. + """ + + def __init__(self, name: str) -> None: + super().__init__(name=name) + # Store the root of this apps for later + this_file = Path(__file__).resolve() + self._this_root = this_file.parent + + def grab_files_step(self) -> None: + """ + Grabs the required source files and optimisation scripts. + """ + super().grab_files_step() + dirs = ['applications/skeleton/source/'] + + # pylint: disable=redefined-builtin + for dir in dirs: + grab_folder(self.config, src=self.lfric_core_root / dir, + dst_label='') + + # Copy the optimisation scripts into a separate directory + grab_folder(self.config, src=self._this_root / "optimisation", + dst_label='optimisation') + + def get_rose_meta(self) -> Path: + """ + :returns: the rose-meta.conf path. + """ + return (self._this_root / 'rose-meta' / 'lfric-skeleton' / 'HEAD' / + 'rose-meta.conf') + + +# ----------------------------------------------------------------------------- +if __name__ == '__main__': + + logger = logging.getLogger('fab') + logger.setLevel(logging.DEBUG) + fab_skeleton = FabSkeleton(name="skeleton") + fab_skeleton.build() diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py new file mode 100755 index 000000000..0cfdc0acc --- /dev/null +++ b/infrastructure/build/fab/configurator.py @@ -0,0 +1,90 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +""" +This file defines the configurator script sequence for LFRic. +""" + +import logging +from pathlib import Path +from typing import cast, Optional + +from fab.api import BuildConfig, find_source_files, Category +from fab.tools.shell import Shell + +from rose_picker_tool import RosePicker + +logger = logging.getLogger('fab') + + +def configurator(config: BuildConfig, + lfric_core_source: Path, + rose_meta_conf: Path, + rose_picker: RosePicker, + include_paths: Optional[list[Path]] = None, + config_dir: Optional[Path] = None) -> None: + """ + This method implements the LFRic configurator tool. + + :param config: the Fab build config instance + :param lfric_core_source: the path to the LFRic core directory + :param rose_meta_conf: the path to the rose-meta configuration file + :param rose_picker: the rose picker tool + :param include_paths: additional include paths (each path will be added, + as well as the path with /'rose-meta') + :param config_dir: the directory for the generated configuration files + """ + + tools = lfric_core_source / 'infrastructure' / 'build' / 'tools' + config_dir = config_dir or config.build_output / 'configuration' + config_dir.mkdir(parents=True, exist_ok=True) + + # rose picker + # ----------- + # creates rose-meta.json and config_namelists.txt in + # gungho/build + logger.info('rose_picker') + + include_dirs = [lfric_core_source, lfric_core_source / 'rose-meta'] + if include_paths: + for path in include_paths: + include_dirs.extend([path, path / 'rose-meta']) + + parameters = [rose_meta_conf, '-directory', config_dir] + for incl_dir in include_dirs: + parameters.extend(['-include_dirs', incl_dir]) + + rose_picker.execute(parameters=parameters) + rose_meta = config_dir / 'rose-meta.json' + + shell = config.tool_box.get_tool(Category.SHELL) + shell = cast(Shell, shell) + + # build_config_loaders + # -------------------- + # builds a bunch of f90s from the json + logger.info('GenerateNamelist') + shell.exec(f"{tools / 'GenerateNamelist'} -verbose {rose_meta} " + f"-directory {config_dir}") + + # create configuration_mod.f90 in source root + # ------------------------------------------- + logger.info('GenerateLoader') + with open(config_dir / 'config_namelists.txt', encoding="utf8") as f_in: + names = [name.strip() for name in f_in.readlines()] + + configuration_mod_fpath = config_dir / 'configuration_mod.f90' + shell.exec(f"{tools / 'GenerateLoader'} {configuration_mod_fpath} " + f"{' '.join(names)}") + + # create feign_config_mod.f90 in source root + # ------------------------------------------ + logger.info('GenerateFeigns') + feign_config_mod_fpath = config_dir / 'feign_config_mod.f90' + shell.exec(f"{tools / 'GenerateFeigns'} {rose_meta} " + f"-output {feign_config_mod_fpath}") + + find_source_files(config, source_root=config_dir) diff --git a/infrastructure/build/fab/rose_picker_tool.py b/infrastructure/build/fab/rose_picker_tool.py new file mode 100755 index 000000000..d0dbb638a --- /dev/null +++ b/infrastructure/build/fab/rose_picker_tool.py @@ -0,0 +1,113 @@ +#!/usr/bin/python3 + +'''This module contains a function that returns a working version of a +rose_picker tool. It can either be a version installed in the system, +or otherwise a checked-out version in the fab-workspace will be used. +If required, a version of rose_picker will be checked out. +''' + +import logging +import os +from pathlib import Path +import shutil +from typing import cast, List, Union + +from fab.api import Category, Tool, ToolRepository +from fab.tools.versioning import Fcm +from fab.util import get_fab_workspace + +logger = logging.getLogger('fab') + + +class RosePicker(Tool): + '''This implements rose_picker as a Fab tool. It supports dynamically + adding the required PYTHONPATH to the environment in case that rose_picker + is not installed, but downloaded. + + :param Path path: the path to the rose picker binary. + ''' + def __init__(self, path: Path): + super().__init__("rose_picker", exec_name=str(path)) + # This is the required PYTHONPATH for running rose_picker + # when it is installed from the repository: + self._pythonpath = path.parents[1] / "lib" / "python" + + def check_available(self) -> bool: + ''' + :returns bool: whether rose_picker works by running + `rose_picker -help`. + ''' + try: + self.run(additional_parameters="-help") + except RuntimeError: + return False + + return True + + def execute(self, parameters: List[Union[Path, str]]) -> None: + ''' + This wrapper adds the required PYTHONPATH, and passes all + parameters through to the tool's run function. + + :param additional_parameter: A list of parameters for rose picker. + ''' + env = os.environ.copy() + env["PYTHONPATH"] = (f"{env.get('PYTHONPATH', '')}:" + f"{self._pythonpath}") + + self.run(additional_parameters=parameters, env=env) + + +# ============================================================================= +def get_rose_picker(tag: str = "v2.0.0") -> RosePicker: + ''' + Returns a Fab RosePicker tool. It can either be a version installed + in the system, which is requested by setting tag to `system`, or a + newly installed version via an FCM checkout. If there is already a + checked-out version, it will be used (i.e. no repeated downloads are + done). + + :param tag: Either the tag in the repository to use, + or 'system' to indicate to use a version installed in the system. + + :returns RosePicker: a Fab RosePicker tool instance + ''' + + if tag.lower() == "system": + # 'system' means to use a rose_picker installed in the system + which_rose_picker = shutil.which("rose_picker") + if not which_rose_picker: + raise RuntimeError("Cannot find system rose_picker tool.") + return RosePicker(Path(which_rose_picker)) + + # Otherwise use rose_picker from the default Fab workspace. It will + # create a instance of the class above, which will add its path to + # PYTHONPATH when executing a rose_picker command. + + gpl_utils = get_fab_workspace() / f"gpl-utils-{tag}" / "source" + rp_path = gpl_utils / "bin" / "rose_picker" + rp = RosePicker(rp_path) + + # If the tool is not available (the class will run `rose_picker -help` + # to verify this ), install it + if not rp.is_available: + fcm = ToolRepository().get_default(Category.FCM) + fcm = cast(Fcm, fcm) + # TODO: atm we are using fcm for the checkout, because using FCM + # keywords is more portable. We cannot use a Fab config (since this + # function is called from within a Fab build), so that means the + # gpl-utils-* directories in the Fab workspace directories do not + # have the normal directory layout. + logger.info(f"Installing rose_picker tag '{tag}'.") + fcm.checkout(src=f'fcm:lfric_gpl_utils.x/tags/{tag}', + dst=gpl_utils) + + # We need to create a new instance, since `is_available` is + # cached (I.e. it's always false in the previous instance) + rp = RosePicker(rp_path) + + if not rp.is_available: + msg = f"Cannot run rose_picker tag '{tag}'." + logger.exception(msg) + raise RuntimeError(msg) + return rp diff --git a/infrastructure/build/fab/site_specific/default/__init__.py b/infrastructure/build/fab/site_specific/default/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infrastructure/build/fab/site_specific/default/config.py b/infrastructure/build/fab/site_specific/default/config.py new file mode 100644 index 000000000..74f1291bd --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/config.py @@ -0,0 +1,163 @@ +#! /usr/bin/env python3 + + +''' +This module contains the default Baf configuration class. +''' + +import argparse +from typing import List + +from fab.api import AddFlags, BuildConfig, Category, ToolRepository + +from default.setup_cray import setup_cray +from default.setup_gnu import setup_gnu +from default.setup_intel_classic import setup_intel_classic +from default.setup_intel_llvm import setup_intel_llvm +from default.setup_nvidia import setup_nvidia + + +class Config: + ''' + This class is the default Configuration object for Baf builds. + It provides several callbacks which will be called from the build + scripts to allow site-specific customisations. + ''' + + def __init__(self): + self._args = None + + @property + def args(self) -> argparse.Namespace: + ''' + :returns argparse.Namespace: the command line options specified by + the user. + ''' + return self._args + + def get_valid_profiles(self) -> List[str]: + ''' + Determines the list of all allowed compiler profiles. The first + entry in this list is the default profile to be used. This method + can be overwritten by site configs to add or modify the supported + profiles. + + :returns List[str]: list of all supported compiler profiles. + ''' + return ["full-debug", "fast-debug", "production", "unit-tests"] + + def update_toolbox(self, build_config: BuildConfig) -> None: + ''' + Set the default compiler flags for the various compiler + that are supported. + + :param build_config: the Fab build configuration instance + ''' + # First create the default compiler profiles for all available + # compilers. While we have a tool box with exactly one compiler + # in it, compiler wrappers will require more than one compiler + # to be initialised - so we just initialise all of them (including + # the linker): + tr = ToolRepository() + for compiler in (tr[Category.C_COMPILER] + + tr[Category.FORTRAN_COMPILER] + + tr[Category.LINKER]): + # Define a base profile, which contains the common + # compilation flags. This 'base' is not accessible to + # the user, so it's not part of the profile list. Also, + # make it inherit from the default profile '', so that + # a user does not have to specify the "base" profile. + # Note that we set this even if a compiler is not available. + # This is required in case that compilers are not in PATH, + # so e.g. mpif90-ifort works, but ifort cannot be found. + # We still need to be able to set and query flags for ifort. + compiler.define_profile("base", inherit_from="") + for profile in self.get_valid_profiles(): + compiler.define_profile(profile, inherit_from="base") + + self.setup_intel_classic(build_config) + self.setup_intel_llvm(build_config) + self.setup_gnu(build_config) + self.setup_nvidia(build_config) + self.setup_cray(build_config) + + def handle_command_line_options(self, args: argparse.Namespace) -> None: + ''' + Additional callback function executed once all command line + options have been added. This is for example used to add + Vernier profiling flags, which are site-specific. + + :param argparse.Namespace args: the command line options added in + the site configs + ''' + # Keep a copy of the args, so they can be used when + # initialising compilers + self._args = args + + def setup_cray(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Cray compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + :type build_config: :py:class:`fab.BuildConfig` + ''' + setup_cray(build_config, self.args) + + def setup_gnu(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Gnu compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + :type build_config: :py:class:`fab.BuildConfig` + ''' + setup_gnu(build_config, self.args) + + def setup_intel_classic(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Intel classic compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + :type build_config: :py:class:`fab.BuildConfig` + ''' + setup_intel_classic(build_config, self.args) + + def setup_intel_llvm(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Intel LLVM compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + :type build_config: :py:class:`fab.BuildConfig` + ''' + setup_intel_llvm(build_config, self.args) + + def setup_nvidia(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Nvidia compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + :type build_config: :py:class:`fab.BuildConfig` + ''' + setup_nvidia(build_config, self.args) + + def get_path_flags(self, build_config: BuildConfig) -> List[AddFlags]: + ''' + Returns the path-specific flags to be used. + TODO FAB #313: Ideally we have only one kind of flag, but as a quick + work around we provide this method. + ''' + return [] diff --git a/infrastructure/build/fab/site_specific/default/setup_cray.py b/infrastructure/build/fab/site_specific/default/setup_cray.py new file mode 100644 index 000000000..346b452fe --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/setup_cray.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +''' +This file contains a function that sets the default flags for the Cray +compilers and linkers in the ToolRepository. + +This function gets called from the default site-specific config file +''' + +import argparse +from typing import cast + +from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository + + +def setup_cray(build_config: BuildConfig, args: argparse.Namespace) -> None: + # pylint: disable=unused-argument + ''' + Defines the default flags for ftn. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :param args: all command line options + ''' + + tr = ToolRepository() + ftn = tr.get_tool(Category.FORTRAN_COMPILER, "crayftn-ftn") + ftn = cast(Compiler, ftn) + + if not ftn.is_available: + return + + # The base flags + # ============== + flags = ["-g", "-G0", "-m", "0", # ? + "-M", "E664,E7208,E7212", # ? + "-en", # Fortran standard + "-ef", # use lowercase module names!Important! + "-hnocaf", # Required for linking with C++ + ] + + # Handle accelerator options: + if args.openacc or args.openmp: + host = args.host.lower() + else: + # Neither openacc nor openmp specified + host = "" + + if args.openacc: + if host == "gpu": + flags.extend(["-h acc"]) + else: + # CPU + flags.extend(["-h acc"]) + elif args.openmp: + if host == "gpu": + flags.extend([]) + else: + # OpenMP on CPU, that's already handled by Fab + pass + + ftn.add_flags(flags, "base") + + # Full debug + # ========== + ftn.add_flags(["-Ktrap=fp", # floating point checking + "-R", "bcdps", # bounds, array shape, collapse, + # pointer, string checking + "-O0"], # No optimisation + "full-debug") + if ftn.get_version() >= (15, 0): + ftn.add_flags(["-G0"], "full-debug") + else: + ftn.add_flags(["-Gfast"], "full-debug") + + # Fast debug + # ========== + ftn.add_flags(["-O2", "-hflex_mp=strict"], "fast-debug") + if ftn.get_version() >= (15, 0): + ftn.add_flags(["-G2"], "fast-debug") + else: + ftn.add_flags(["-Gfast"], "fast-debug") + + # Production + # ========== + ftn.add_flags(["-O3", "-hipa3", "-m", "3"], "production") + + # Set up the linker + # ================= + linker = tr.get_tool(Category.LINKER, f"linker-{ftn.name}") + linker = cast(Linker, linker) + + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + + try: + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + except RuntimeError: + nc_flibs = [] + + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + linker.add_post_lib_flags(["-lcraystdc++"]) + + # Using the GNU compiler on Crays for now needs the additional + # flag -fallow-argument-mismatch to compile mpi_mod.f90 + ftn = tr.get_tool(Category.FORTRAN_COMPILER, "crayftn-gfortran") + ftn.add_flags("-fallow-argument-mismatch") diff --git a/infrastructure/build/fab/site_specific/default/setup_gnu.py b/infrastructure/build/fab/site_specific/default/setup_gnu.py new file mode 100644 index 000000000..c00c99b91 --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/setup_gnu.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +''' +This file contains a function that sets the default flags for all +GNU based compilers and linkers in the ToolRepository. + +This function gets called from the default site-specific config file +''' + +import argparse +from typing import cast + +from fab.api import BuildConfig, Category, Linker, ToolRepository + + +def setup_gnu(build_config: BuildConfig, args: argparse.Namespace) -> None: + # pylint: disable=unused-argument + ''' + Defines the default flags for all GNU compilers and linkers. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :param args: all command line options + ''' + + tr = ToolRepository() + gfortran = tr.get_tool(Category.FORTRAN_COMPILER, "gfortran") + + if not gfortran.is_available: + gfortran = tr.get_tool(Category.FORTRAN_COMPILER, "mpif90-gfortran") + if not gfortran.is_available: + return + + if gfortran.get_version() < (4, 9): + raise RuntimeError(f"GFortran is too old to build dynamo. " + f"Must be at least 4.9.0, it is " + f"'{gfortran.get_version_string()}'.") + + # The base flags + # ============== + + # TODO: It should use -Werror=conversion, but: + # Most lfric_atm dependencies contain code with implicit lossy + # conversions. + # This should be restricted to only the files/directories + # that need it, but this needs Fab updates. + + gfortran.add_flags( + ['-ffree-line-length-none', '-Wall', '-g', + '-Werror=character-truncation', + '-Werror=unused-value', + '-Werror=tabs', + '-std=f2008', + '-fdefault-real-8', + '-fdefault-double-8', + ], + "base") + + # TODO - Remove the -fallow-arguments-mismatch flag when MPICH no longer + # fails to build as a result of its mismatched arguments (see + # ticket summary for #2549 for reasoning). + if gfortran.get_version() >= (10, 0): + gfortran.add_flags("-fallow-argument-mismatch", "base") + + runtime = ["-fcheck=all", "-ffpe-trap=invalid,zero,overflow"] + init = ["-finit-integer=31173", "-finit-real=snan", + "-finit-logical=true", "-finit-character=85"] + # Full debug + # ========== + gfortran.add_flags(runtime + ["-O0"] + init, "full-debug") + + # Fast debug + # ========== + gfortran.add_flags(runtime + ["-Og"], "fast-debug") + + # Production + # ========== + gfortran.add_flags(["-Ofast"], "production") + + # unit-tests + # ========== + gfortran.add_flags(runtime + ["-O0"] + init, "unit-tests") + + # Set up the linker + # ================= + # This will implicitly affect all gfortran based linkers, e.g. + # linker-mpif90-gfortran will use these flags as well. + linker = tr.get_tool(Category.LINKER, f"linker-{gfortran.name}") + linker = cast(Linker, linker) + + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + + try: + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + except RuntimeError: + nc_flibs = [] + + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + # Always link with C++ libs + linker.add_post_lib_flags(["-lstdc++"], "base") diff --git a/infrastructure/build/fab/site_specific/default/setup_intel_classic.py b/infrastructure/build/fab/site_specific/default/setup_intel_classic.py new file mode 100644 index 000000000..ab451f30d --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/setup_intel_classic.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +''' +This file contains a function that sets the default flags for all +Intel classic based compilers in the ToolRepository (ifort, icc). + +This function gets called from the default site-specific config file +''' + +import argparse +from typing import cast + +from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository + + +def setup_intel_classic(build_config: BuildConfig, + args: argparse.Namespace) -> None: + # pylint: disable=unused-argument, too-many-locals + ''' + Defines the default flags for all Intel classic compilers and linkers. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :param args: all command line options + ''' + + tr = ToolRepository() + ifort = tr.get_tool(Category.FORTRAN_COMPILER, "ifort") + ifort = cast(Compiler, ifort) + + if not ifort.is_available: + # This can happen if ifort is not in path (in spack environments). + # To support this common use case, see if mpif90-ifort is available, + # and initialise this otherwise. + ifort = tr.get_tool(Category.FORTRAN_COMPILER, "mpif90-ifort") + ifort = cast(Compiler, ifort) + if not ifort.is_available: + # Since some flags depends on version, the code below requires + # that the intel compiler actually works. + return + + # The base flags + # ============== + # The following flags will be applied to all modes: + ifort.add_flags(["-stand", "f08"], "base") + ifort.add_flags(["-g", "-traceback"], "base") + # With -warn errors we get externals that are too long. While this + # is a (usually safe) warning, the long externals then causes the + # build to abort. So for now we cannot use `-warn errors` + ifort.add_flags(["-warn", "all"], "base") + + # By default turning interface warnings on causes "genmod" files to be + # created. This adds unnecessary files to the build so we disable that + # behaviour. + ifort.add_flags(["-gen-interfaces", "nosource"], "base") + + # The "-assume realloc-lhs" switch causes Intel Fortran prior to v17 to + # actually implement the Fortran2003 standard. At version 17 it becomes the + # default behaviour. + if ifort.get_version() < (17, 0): + ifort.add_flags(["-assume", "realloc-lhs"], "base") + + # Full debug + # ========== + # ifort.mk: bad interaction between array shape checking and + # the matmul" intrinsic in at least some iterations of v19. + if (19, 0, 0) <= ifort.get_version() < (19, 1, 0): + runtime_flags = ["-check", "all,noshape", "-fpe0"] + else: + runtime_flags = ["-check", "all", "-fpe0"] + ifort.add_flags(runtime_flags, "full-debug") + ifort.add_flags(["-O0", "-ftrapuv"], "full-debug") + + # Fast debug + # ========== + ifort.add_flags(["-O2", "-fp-model=strict"], "fast-debug") + + # Production + # ========== + ifort.add_flags(["-O3", "-xhost"], "production") + + # Set up the linker + # ================= + # This will implicitly affect all ifort based linkers, e.g. + # linker-mpif90-ifort will use these flags as well. + linker = tr.get_tool(Category.LINKER, f"linker-{ifort.name}") + linker = cast(Linker, linker) + + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + try: + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + except RuntimeError: + nc_flibs = [] + + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + # Always link with C++ libs + linker.add_post_lib_flags(["-lstdc++"]) diff --git a/infrastructure/build/fab/site_specific/default/setup_intel_llvm.py b/infrastructure/build/fab/site_specific/default/setup_intel_llvm.py new file mode 100644 index 000000000..d3a99bb68 --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/setup_intel_llvm.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +''' +This file contains a function that sets the default flags for all +Intel llvm based compilers and linkers in the ToolRepository (ifx, icx). + +This function gets called from the default site-specific config file +''' + +import argparse +from typing import cast + +from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository + + +def setup_intel_llvm(build_config: BuildConfig, + args: argparse.Namespace) -> None: + # pylint: disable=unused-argument, too-many-locals + ''' + Defines the default flags for all Intel llvm compilers. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :param args: all command line options + ''' + + tr = ToolRepository() + ifx = tr.get_tool(Category.FORTRAN_COMPILER, "ifx") + ifx = cast(Compiler, ifx) + + if not ifx.is_available: + ifx = tr.get_tool(Category.FORTRAN_COMPILER, "mpif90-ifx") + ifx = cast(Compiler, ifx) + if not ifx.is_available: + return + + # The base flags + # ============== + # The following flags will be applied to all modes: + ifx.add_flags(["-stand", "f08"], "base") + ifx.add_flags(["-g", "-traceback"], "base") + # With -warn errors we get externals that are too long. While this + # is a (usually safe) warning, the long externals then causes the + # build to abort. So for now we cannot use `-warn errors` + ifx.add_flags(["-warn", "all"], "base") + + # By default turning interface warnings on causes "genmod" files to be + # created. This adds unnecessary files to the build so we disable that + # behaviour. + ifx.add_flags(["-gen-interfaces", "nosource"], "base") + + # Full debug + # ========== + ifx.add_flags(["-check", "all", "-fpe0"], "full-debug") + ifx.add_flags(["-O0", "-ftrapuv"], "full-debug") + + # Fast debug + # ========== + ifx.add_flags(["-O2", "-fp-model=strict"], "fast-debug") + + # Production + # ========== + ifx.add_flags(["-O3", "-xhost"], "production") + + # Set up the linker + # ================= + # This will implicitly affect all ifx based linkers, e.g. + # linker-mpif90-ifx will use these flags as well. + linker = tr.get_tool(Category.LINKER, f"linker-{ifx.name}") + linker = cast(Linker, linker) # Make mypy happy + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + try: + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + except RuntimeError: + nc_flibs = [] + + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + # Always link with C++ libs + linker.add_post_lib_flags(["-lstdc++"]) diff --git a/infrastructure/build/fab/site_specific/default/setup_nvidia.py b/infrastructure/build/fab/site_specific/default/setup_nvidia.py new file mode 100644 index 000000000..63b179a79 --- /dev/null +++ b/infrastructure/build/fab/site_specific/default/setup_nvidia.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +''' +This file contains a function that sets the default flags for the NVIDIA +compilers and linkers in the ToolRepository. + +This function gets called from the default site-specific config file +''' + +import argparse +from typing import cast + +from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository + + +def setup_nvidia(build_config: BuildConfig, args: argparse.Namespace) -> None: + # pylint: disable=unused-argument + ''' + Defines the default flags for nvfortran. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :param args: all command line options + ''' + + tr = ToolRepository() + nvfortran = tr.get_tool(Category.FORTRAN_COMPILER, "nvfortran") + nvfortran = cast(Compiler, nvfortran) + + if not nvfortran.is_available: + nvfortran = tr.get_tool(Category.FORTRAN_COMPILER, "mpif90-nvfortran") + nvfortran = cast(Compiler, nvfortran) + if not nvfortran.is_available: + return + + # The base flags + # ============== + flags = ["-Mextend", # 132 characters line length + "-g", "-traceback", + "-r8", # Default 8 bytes reals + "-O0", # No optimisations + ] + + lib_flags = ["-c++libs"] + + # Handle accelerator options: + if args.openacc or args.openmp: + host = args.host.lower() + else: + # Neither openacc nor openmp specified + host = "" + + if args.openacc: + if host == "gpu": + flags.extend(["-acc=gpu", "-gpu=managed"]) + lib_flags.extend(["-aclibs", "-cuda"]) + else: + # CPU + flags.extend(["-acc=cpu"]) + elif args.openmp: + if host == "gpu": + flags.extend(["-mp=gpu", "-gpu=managed"]) + lib_flags.append("-cuda") + else: + # OpenMP on CPU, that's already handled by Fab + pass + + nvfortran.add_flags(flags, "base") + + # Full debug + # ========== + nvfortran.add_flags(["-O0", "-fp-model=strict"], "full-debug") + + # Fast debug + # ========== + nvfortran.add_flags(["-O2", "-fp-model=strict"], "fast-debug") + + # Production + # ========== + nvfortran.add_flags(["-O4"], "production") + + # Set up the linker + # ================= + # This will implicitly affect all nvfortran based linkers, e.g. + # linker-mpif90-nvfortran will use these flags as well. + linker = tr.get_tool(Category.LINKER, f"linker-{nvfortran.name}") + linker = cast(Linker, linker) + + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + try: + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + except RuntimeError: + nc_flibs = [] + + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + # Always link with C++ libs + linker.add_post_lib_flags(lib_flags) diff --git a/infrastructure/build/fab/site_specific/meto_ex1a/config.py b/infrastructure/build/fab/site_specific/meto_ex1a/config.py new file mode 100644 index 000000000..c62925243 --- /dev/null +++ b/infrastructure/build/fab/site_specific/meto_ex1a/config.py @@ -0,0 +1,42 @@ +#! /usr/bin/env python3 + +'''This module contains a setup for METO-EX1A +''' + +from typing import cast + +from fab.api import BuildConfig, Category, Linker, ToolRepository + +from default.config import Config as DefaultConfig + + +class Config(DefaultConfig): + '''This config class sets specific flags for METO-EX1A + ''' + + def __init__(self): + super().__init__() + tr = ToolRepository() + # Set cray as default compiler suite + # It has crayftn-ftn as Fortran compiler + # It also has craycc-cc as C compiler + tr.set_default_compiler_suite("cray") + + def setup_cray(self, build_config: BuildConfig): + '''First call the base class to get all default options. + See the file ../default/setup_cray.py for the current + default. + Very likely, linker options need to be changed: + ''' + super().setup_cray(build_config) + tr = ToolRepository() + + # Update the linker. This is what the default sets up + # (except NetCDF, which is normally defined using nf-config) + linker = tr.get_tool(Category.LINKER, "linker-crayftn-ftn") + linker = cast(Linker, linker) # make mypy happy + + # Don't know whether Cray uses nf-config. So hard-code + # these flags for now until the transition to pkg-config + linker.add_lib_flags("netcdf", ["-lnetcdff", "-lnetcdf", + "-lnetcdf", "-lm"]) diff --git a/infrastructure/build/fab/site_specific/ncas_ex/config.py b/infrastructure/build/fab/site_specific/ncas_ex/config.py new file mode 100644 index 000000000..7d287e5ee --- /dev/null +++ b/infrastructure/build/fab/site_specific/ncas_ex/config.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python3 + +'''This module contains a setup for NCAS-EX (archer2) +''' + +from typing import cast + +from fab.api import BuildConfig, Category, Linker, ToolRepository + +from default.config import Config as DefaultConfig + + +class Config(DefaultConfig): + '''This config class sets specific flags for NCAS-EX (archer2) + ''' + + def __init__(self): + super().__init__() + tr = ToolRepository() + tr.set_default_compiler_suite("gnu") + + def setup_cray(self, build_config: BuildConfig): + '''First call the base class to get all default options. + See the file ../default/setup_cray.py for the current + default. + Very likely, linker options need to be changed: + ''' + super().setup_cray(build_config) + tr = ToolRepository() + ftn = tr.get_tool(Category.FORTRAN_COMPILER, "gfortran") + # Any gfortran on Cray's EX need this flag in order to + # compile mpi_mod: + ftn.add_flags(["-fallow-argument-mismatch"]) + + # Update the linker. This is what the default sets up + # (except NetCDF, which is defined using nf-config, and + # should likely work the way it is): + linker = tr.get_tool(Category.LINKER, "linker-gfortran") + linker = cast(Linker, linker) # make mypy happy + + # Cray's don't have nf-config. Till we have figured out the + # proper solution, hard-code some flags that might work + # with a plain gfortran build in a spack environment: + linker.add_lib_flags("netcdf", ["-lnetcdff", "-lnetcdf", + "-lnetcdf", "-lm"]) + # That's pretty much the default: + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) diff --git a/infrastructure/build/fab/site_specific/nci_gadi/__init__.py b/infrastructure/build/fab/site_specific/nci_gadi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infrastructure/build/fab/site_specific/nci_gadi/config.py b/infrastructure/build/fab/site_specific/nci_gadi/config.py new file mode 100644 index 000000000..cf826774f --- /dev/null +++ b/infrastructure/build/fab/site_specific/nci_gadi/config.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python3 + +''' +This module contains the default configuration for NCI. It will be invoked +by the Baf scripts. This script: +- sets intel-classic as the default compiler suite to use. +- Adds the tau compiler wrapper as (optional) compilers to the ToolRepository. +''' + +from pathlib import Path +from typing import List, Union, Optional + +from fab.api import (BuildConfig, Category, Compiler, CompilerWrapper, + ToolRepository) + +from default.config import Config as DefaultConfig + + +class Tauf90(CompilerWrapper): + ''' + Class for the Tau profiling Fortran compiler wrapper. + It will be using the name "tau-COMPILER_NAME", but will call tau_f90.sh. + + :param compiler: the compiler that the tau_f90.sh wrapper will use. + :type compiler: :py:class:`fab.tools.Compiler` + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"tau-{compiler.name}", + exec_name="tau_f90.sh", compiler=compiler, mpi=True) + + def compile_file(self, input_file: Path, + output_file: Path, + config: BuildConfig, + add_flags: Union[None, List[str]] = None, + syntax_only: Optional[bool] = None) -> None: + ''' + This method overrides the Fab CompilerWrapper class compile_file + method to fall back to the wrapped compiler for certain Fortran files + and use the tau_f90.sh wrapper to compile the rest. + + :param Path input_file: the path of the input file to compile + :param Path output_file: the path of the output file to create + :param config: the Fab build configuration instance + :type config: :py:class:`fab.BuildConfig` + :param add_flags: additional flags to pass to the compiler + :type add_flags: Union[None, List[str]] + :param syntax_only: whether to only check the syntax of the file + :type syntax_only: Optional[bool] + ''' + if ('psy.f90' in str(input_file)) or \ + ('/kernel/' in str(input_file)) or \ + ('leaf_jls_mod' in str(input_file)) or \ + ('/science/' in str(input_file)): + self.compiler.compile_file(input_file, output_file, + config, add_flags, syntax_only) + else: + super().compile_file(input_file, output_file, + config, add_flags, syntax_only) + + +class Taucc(CompilerWrapper): + ''' + Class for the Tau profiling C compiler wrapper. + It will be using the name "tau-COMPILER_NAME", but will call tau_cc.sh. + + :param compiler: the compiler that the tau_cc.sh wrapper will use + :type compiler: :py:class:`fab.tools.Compiler` + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"tau-{compiler.name}", + exec_name="tau_cc.sh", compiler=compiler, mpi=True) + + +class Config(DefaultConfig): + ''' + For NCI, make intel the default, and add the Tau wrapper. + ''' + + def __init__(self): + super().__init__() + tr = ToolRepository() + tr.set_default_compiler_suite("intel-classic") + + # Add the tau wrappers for Fortran and C. Note that add_tool + # will automatically add them as a linker as well. + for ftn in ["ifort", "gfortran"]: + compiler = tr.get_tool(Category.FORTRAN_COMPILER, ftn) + tr.add_tool(Tauf90(compiler)) + + for cc in ["icc", "gcc"]: + compiler = tr.get_tool(Category.C_COMPILER, cc) + tr.add_tool(Taucc(compiler)) + + # ATM we don't use a shell when running a tool, and as such + # we can't directly use "$()" as parameter. So query these values using + # Fab's shell tool (doesn't really matter which shell we get, so just + # ask for the default): + shell = tr.get_default(Category.SHELL) + # We must remove the trailing new line, and create a list: + nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], + capture_output=True).strip().split() + linker = tr.get_tool(Category.LINKER, "linker-tau-ifort") + linker.add_lib_flags("netcdf", nc_flibs) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) + + # Always link with C++ libs + linker.add_post_lib_flags(["-lstdc++"]) diff --git a/infrastructure/build/fab/site_specific/niwa_xc50/config.py b/infrastructure/build/fab/site_specific/niwa_xc50/config.py new file mode 100644 index 000000000..f9e6008b2 --- /dev/null +++ b/infrastructure/build/fab/site_specific/niwa_xc50/config.py @@ -0,0 +1,61 @@ +#! /usr/bin/env python3 + +''' +This module contains a setup NIWA's XC-50 +''' + +import os +from typing import cast + +from fab.api import BuildConfig, Category, Linker, ToolRepository + +from default.config import Config as DefaultConfig + + +class Config(DefaultConfig): + ''' + This config class sets specific flags for NIWA's XC-50 + ''' + + def __init__(self): + super().__init__() + tr = ToolRepository() + tr.set_default_compiler_suite("intel-classic") + + def setup_cray(self, build_config: BuildConfig) -> None: + ''' + First call the base class to get all default options. + See the file ../default/setup_cray.py for the current + default. Then the NIWA's XC-50 specific flags are added. + The linker is also updated. + + :param build_config: the Fab build config instance from which + required parameters can be taken. + :type build_config: :py:class:`fab.BuildConfig` + ''' + super().setup_cray(build_config) + tr = ToolRepository() + ftn = tr.get_tool(Category.FORTRAN_COMPILER, "crayftn-ifort") + # Add any flags you want to have: + ftn.add_flags([f"-I{os.environ['EBROOTXIOS']}/inc"]) + + # Update the linker. This is what the default sets up + # (except NetCDF, which is defined using nf-config, and + # should likely work the way it is): + linker = tr.get_tool(Category.LINKER, "linker-crayftn-ftn") + linker = cast(Linker, linker) # make mypy happy + + # The first parameter specifies the internal name for libraries, + # followed by a list of linker options. If you should need additional + # library paths, you could e.g. use: + # linker.add_lib_flags("yaxt", ["-L", "/my/path/to/yaxt", "-lyaxt", + # "-lyaxt_c"]) + # Make sure to not use a space as ONE parameter ("-L /my/lib"), + # you have to specify them as two separate list elements + + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) + linker.add_lib_flags("xios", ["-lxios"]) + linker.add_lib_flags("hdf5", ["-lhdf5"]) + linker.add_lib_flags("shumlib", ["-lshum"]) + linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", + "-lvernier"]) diff --git a/infrastructure/build/fab/templaterator.py b/infrastructure/build/fab/templaterator.py new file mode 100755 index 000000000..94a97115a --- /dev/null +++ b/infrastructure/build/fab/templaterator.py @@ -0,0 +1,58 @@ +#!/usr/bin/python3 + +''' +This module contains the Fab Templaterator class. +''' + +import logging +from pathlib import Path +from typing import Dict, List, Union + +from fab.api import Tool + +logger = logging.getLogger('fab') + + +class Templaterator(Tool): + '''This implements the LFRic templaterator as a Fab tool. + It can check whether the templaterator is available and + creates command line options for it to run. + + :param Path exec_name: the path to the templaterator binary. + ''' + def __init__(self, exec_name: Path): + super().__init__(exec_name.name, exec_name=str(exec_name)) + + def check_available(self) -> bool: + ''' + :returns bool: whether templaterator works by running + `Templaterator -help`. + ''' + try: + super().run(additional_parameters="-h") + except RuntimeError: + return False + + return True + + def process(self, input_template: Path, + output_file: Path, + key_values: Dict[str, str]) -> None: + """ + This wrapper runs the Templaterator, which replaces the + give keys in the input template with the value in the + `key_values` dictionary. The new file is written to the + specified output file. + + :param input_template: the path to the input template. + :param output_file: the output file path. + :param key_values: the keys and values for the keys to + define as a dictionary. + """ + replace_list = [f"-s {key}={value}" + for key, value in key_values.items()] + params: List[Union[str, Path]] + params = [input_template, "-o", output_file] + params.extend(replace_list) + + super().run(additional_parameters=params) From a350ef3e938bf814d12a6de419a55654b93de94d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 12:49:37 +1100 Subject: [PATCH 04/95] #240 Updated tests to pass all linters, fixed licence statement. --- infrastructure/build/fab/configurator.py | 6 +- infrastructure/build/fab/lfric_base.py | 21 +- infrastructure/build/fab/rose_picker_tool.py | 15 +- infrastructure/build/fab/templaterator.py | 12 +- .../build/fab/test/configurator_test.py | 101 +++ .../build/fab/test/lfric_base_test.py | 752 ++++++++++++++++++ .../build/fab/test/rose_picker_tool_test.py | 125 +++ .../build/fab/test/templaterator_test.py | 76 ++ 8 files changed, 1090 insertions(+), 18 deletions(-) create mode 100644 infrastructure/build/fab/test/configurator_test.py create mode 100644 infrastructure/build/fab/test/lfric_base_test.py create mode 100644 infrastructure/build/fab/test/rose_picker_tool_test.py create mode 100644 infrastructure/build/fab/test/templaterator_test.py diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index 0cfdc0acc..1c40f383a 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -1,8 +1,10 @@ ############################################################################## # (c) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT -# which you should have received as part of this distribution +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. ############################################################################## +# Author J. Henrichs, Bureau of Meteorology +# Author J. Lyu, Bureau of Meteorology """ This file defines the configurator script sequence for LFRic. diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index fef3876ae..57744132b 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python3 -# ############################################################################## -# (c) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT -# which you should have received as part of this distribution -# ############################################################################## - -''' +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Henrichs, Bureau of Meteorology +# Author: J. Lyu, Bureau of Meteorology + +""" This is an OO basic interface to FAB. It allows the typical LFRic applications to only modify very few settings to have a working FAB build script. -''' +""" import argparse import os @@ -306,7 +307,7 @@ def templaterator_step(self, config: BuildConfig) -> None: config.artefact_store.add(ArtefactSet.FORTRAN_COMPILER_FILES, out_file) - def get_rose_meta(self) -> Union[Path, None]: + def get_rose_meta(self) -> Optional[Path]: ''' This method returns the path to the rose meta data config file. Currently, it returns none. It's up to the LFRic applications to diff --git a/infrastructure/build/fab/rose_picker_tool.py b/infrastructure/build/fab/rose_picker_tool.py index d0dbb638a..202af1c07 100755 --- a/infrastructure/build/fab/rose_picker_tool.py +++ b/infrastructure/build/fab/rose_picker_tool.py @@ -1,10 +1,17 @@ -#!/usr/bin/python3 - -'''This module contains a function that returns a working version of a +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author J. Henrichs, Bureau of Meteorology +# Author J. Lyu, Bureau of Meteorology + +""" +This module contains a function that returns a working version of a rose_picker tool. It can either be a version installed in the system, or otherwise a checked-out version in the fab-workspace will be used. If required, a version of rose_picker will be checked out. -''' +""" import logging import os diff --git a/infrastructure/build/fab/templaterator.py b/infrastructure/build/fab/templaterator.py index 94a97115a..655b9566d 100755 --- a/infrastructure/build/fab/templaterator.py +++ b/infrastructure/build/fab/templaterator.py @@ -1,4 +1,10 @@ -#!/usr/bin/python3 +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Henrichs, Bureau of Meteorology +# Author: J. Lyu, Bureau of Meteorology ''' This module contains the Fab Templaterator class. @@ -21,7 +27,9 @@ class Templaterator(Tool): :param Path exec_name: the path to the templaterator binary. ''' def __init__(self, exec_name: Path): - super().__init__(exec_name.name, exec_name=str(exec_name)) + # Remove suffix as a name + super().__init__(exec_name.stem, + exec_name=exec_name) def check_available(self) -> bool: ''' diff --git a/infrastructure/build/fab/test/configurator_test.py b/infrastructure/build/fab/test/configurator_test.py new file mode 100644 index 000000000..0bb3f0373 --- /dev/null +++ b/infrastructure/build/fab/test/configurator_test.py @@ -0,0 +1,101 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author J. Henrichs, Bureau of Meteorology + +""" +This module tests the configurator. +""" + +from unittest.mock import MagicMock + +import pytest + +from configurator import configurator # Replace with actual module name +from fab.tools.category import Category +from fab.tools.tool_box import ToolBox +from fab.build_config import BuildConfig + + +@pytest.fixture +def mock_shell(): + """ + A simple shell mock to check that all expected calls are executed. + """ + shell = MagicMock() + shell.exec = MagicMock() + # Set the category to the shell can be added to the ToolBox. + shell.category = Category.SHELL + return shell + + +def test_configurator_runs_expected_sequence(mock_shell, tmp_path): + """ + Check that the expected series of calls is executed. + """ + + # Create a tool box and add the mocked shell that is used + # to test the expected calls. + tb = ToolBox() + tb.add_tool(mock_shell) + config = BuildConfig("Stub config", tb, + fab_workspace=tmp_path / 'fab') + + # Create a rose_picker mock: + rose_picker = MagicMock() + rose_picker.execute = MagicMock() + + # Setup source directories and rose-meta config file + lfric_core = tmp_path / "lfric_core" + lfric_apps = tmp_path / "lfric_apps" + rose_meta_conf = tmp_path / "rose-meta.conf" + + # Simulate rose-meta.json and config_namelists.txt creation + config_dir = tmp_path / "build_output" / "configuration" + config_dir.mkdir(parents=True) + rose_meta = config_dir / "rose-meta.json" + rose_meta.write_text("{}", encoding="utf8") + config_namelist = config_dir / "config_namelists.txt" + config_namelist.write_text("namelist1\nnamelist2\n", encoding="utf8") + + # Run configurator + with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): + configurator( + config=config, + lfric_core_source=lfric_core, + rose_meta_conf=rose_meta_conf, + rose_picker=rose_picker, + include_paths=[lfric_apps], + config_dir=config_dir + ) + + tools_dir = lfric_core / "infrastructure" / "build" / "tools" + + # Check rose_picker was called with the expected arguments: + rose_picker.execute.assert_called_once() + kwargs = rose_picker.execute.call_args_list[0].kwargs + assert kwargs["parameters"] == [ + rose_meta_conf, + '-directory', config_dir, + '-include_dirs', lfric_core, + '-include_dirs', lfric_core / "rose-meta", + '-include_dirs', lfric_apps, + '-include_dirs', lfric_apps / "rose-meta" + ] + + # Check shell.exec was called with expected commands + expected_calls = [ + ((f"{tools_dir / 'GenerateNamelist'} " + f"-verbose {config_dir / 'rose-meta.json'} " + f"-directory {config_dir}"),), + ((f"{tools_dir / 'GenerateLoader'} " + f"{config_dir / 'configuration_mod.f90'} " + f"namelist1 namelist2"),), + ((f"{tools_dir / 'GenerateFeigns'} {config_dir / 'rose-meta.json'} " + f"-output {config_dir / 'feign_config_mod.f90'}"),) + ] + + actual_calls = [call.args for call in mock_shell.exec.call_args_list] + assert actual_calls == expected_calls diff --git a/infrastructure/build/fab/test/lfric_base_test.py b/infrastructure/build/fab/test/lfric_base_test.py new file mode 100644 index 000000000..c07279282 --- /dev/null +++ b/infrastructure/build/fab/test/lfric_base_test.py @@ -0,0 +1,752 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Lyu, Bureau of Meteorology +# Author: J. Henrichs, Bureau of Meteorology + +""" +Tests the LFRicBase class. +""" + +from pathlib import Path +import os +import sys +import argparse +import inspect +from unittest import mock +from typing import List, Optional + +import pytest + +from fab.api import (ArtefactSet, BuildConfig, Category, ToolRepository, + Linker) +from fab.tools.compiler import CCompiler, FortranCompiler + +from lfric_base import LFRicBase + + +class MockSiteConfig: + """ + Creates a mock site config class. + """ + def __init__(self) -> None: + self.args: Optional[argparse.Namespace] = None + + def get_valid_profiles(self) -> List[str]: + """ + :return: list of valid compilation profiles. + """ + return ["default-profile"] + + def update_toolbox(self, build_config: BuildConfig) -> None: + """ + Dummy function where the tool box could be modified + """ + + def handle_command_line_options(self, args: argparse.Namespace) -> None: + """ + Simple function to handle command line options. + """ + self.args = args + + def get_path_flags(self, _build_config: BuildConfig) -> List[str]: + """ + :returns: list of path-specific flags. + """ + return [] + + +@pytest.fixture(name="stub_fortran_compiler", scope='function') +def stub_fortran_compiler_init() -> FortranCompiler: + """ + Provides a minimal Fortran compiler. + """ + compiler = FortranCompiler('some Fortran compiler', 'sfc', 'stub', + r'([\d.]+)', openmp_flag='-omp', + module_folder_flag='-mods') + return compiler + + +@pytest.fixture(name="stub_c_compiler", scope='function') +def stub_c_compiler_init() -> CCompiler: + """ + Provides a minimal C compiler. + """ + compiler = CCompiler("some C compiler", "scc", "stub", + version_regex=r"([\d.]+)", openmp_flag='-omp') + return compiler + + +@pytest.fixture(name="stub_linker", scope='function') +def stub_linker_init(stub_c_compiler) -> Linker: + """ + Provides a minimal linker. + """ + linker = Linker(stub_c_compiler, None, 'sln') + return linker + + +@pytest.fixture(scope="function", autouse=True) +def setup_site_specific_config_environment(tmp_path): + """ + This sets up the environment for the mocked site_specific config class + (MockSiteConfig) to be used by tests of LFRicBase class methods without + errors. This fixture is automatically executed for any test in this file. + """ + # Creates mock module with __file__ attribute + mock_site_module = mock.MagicMock() + mock_site_module.Config = MockSiteConfig + mock_site_module.__file__ = str(tmp_path / "site_specific" / + "default" / "config.py") + + # Mocks site-specific imports + sys.modules['site_specific'] = mock.MagicMock() + sys.modules['site_specific.default'] = mock.MagicMock() + sys.modules['site_specific.default.config'] = mock_site_module + + # Clears environment variables + with mock.patch.dict(os.environ, clear=True): + yield + + # Cleanups + for module in ['site_specific', 'site_specific.default', + 'site_specific.default.config']: + if module in sys.modules: + del sys.modules[module] + + +@pytest.fixture(scope="function", autouse=True) +def setup_tool_repository(stub_fortran_compiler, stub_c_compiler, + stub_linker): + ''' + This sets up a ToolRepository that allows the LFRicBase class + to proceed without raising errors. This fixture is automatically + executed for any test in this file. + ''' + # pylint: disable=protected-access + # Make sure we always get a new ToolRepository to not be affected by + # other tests: + ToolRepository._singleton = None + + # Remove all compiler and linker, so we get results independent + # of the software available on the platform this test is running + tr = ToolRepository() + for category in [Category.C_COMPILER, Category.FORTRAN_COMPILER, + Category.LINKER]: + tr[category] = [] + + # Add compilers and linkers, and mark them all as available, + # as well as supporting MPI and OpenMP + for tool in [stub_c_compiler, stub_fortran_compiler, stub_linker]: + tool._mpi = True + tool._openmp_flag = "-some-openmp-flag" + tool._is_available = True + tool._version = (1, 2, 3) + tr.add_tool(tool) + + # Remove environment variables that could affect tests + with mock.patch.dict(os.environ, clear=True): + yield + + # Reset tool repository for other tests + ToolRepository._singleton = None + + +def test_constructor(monkeypatch) -> None: + ''' + Tests constructor. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + lfric_base = LFRicBase(name="test_name") + + # Check root symbol defaults to name if not specified + assert lfric_base.root_symbol == ["test_name"] + + # Check root symbol can be specified + lfric_base = LFRicBase(name="test_name", + root_symbol="root1") + assert lfric_base.root_symbol == ["root1"] + + # Check root symbol list + lfric_base = LFRicBase(name="test_name", + root_symbol=["root1", "root2"]) + assert lfric_base.root_symbol == ["root1", "root2"] + + +def test_get_directory(monkeypatch, tmp_path) -> None: + ''' + Tests the correct setup of lfric_core_root and lfric_apps_root. + ''' + + # Create mock directory structure + mock_core = tmp_path / "core" + mock_core.mkdir(parents=True) + + # Create mock LFRic base file location + mock_base_dir = mock_core / "infrastructure" / "build" / "fab" + mock_base_dir.mkdir(parents=True) + mock_base_file = mock_base_dir / "lfric_base.py" + mock_base_file.write_text("", encoding='utf-8') + + # Mock __file__ attribute + monkeypatch.setattr('lfric_base.__file__', str(mock_base_file)) + + mock_apps = tmp_path / "apps" + mock_apps.mkdir() + deps_file = mock_apps / "dependencies.sh" + deps_file.write_text("", encoding='utf-8') + + mock_caller = mock_apps / "some_app" / "build.py" + mock_caller.parent.mkdir(parents=True) + mock_caller.write_text("", encoding='utf-8') + + # Create mock frame objects with proper structure + def create_frame(filename): + frame = mock.Mock() + frame.f_globals = {'__file__': filename} + return frame + + def create_frame_info(filename): + return (create_frame(filename), filename, None, None, None, None, + None, None) + + # Mock inspect.stack() to return our test callers with proper + # frame info structure + mock_stack = [ + create_frame_info(str(mock_base_file)), # First call in base dir + create_frame_info(str(mock_caller)) # Second call in apps + ] + monkeypatch.setattr('inspect.stack', lambda: mock_stack) + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + lfric_base = LFRicBase(name="test") + + # Verify core root is set correctly + assert lfric_base.lfric_core_root == mock_core + + +def test_command_line_options(monkeypatch) -> None: + ''' + Tests LFRic specific command line options. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py", + "--rose_picker", "custom", + "--precision-default", "32"]) + + lfric_base = LFRicBase(name="test") + + assert lfric_base.args.rose_picker == "custom" + assert lfric_base.args.precision_default == "32" + + +def test_precision_definition_without_default(monkeypatch) -> None: + ''' + Tests specification of precision if no default precision is + specified on the command line (--precision-default). Tests all + other ways a precision can be specified: default command line, + explicit command line, environment variable, and the per + R_*PRECISION default. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py", + "--rdef_precision", "32"]) + monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) + + lfric_base = LFRicBase(name="test") + lfric_base.define_preprocessor_flags_step() + flags = lfric_base.preprocess_flags_common + + # Explicitly set on command line: + assert '-DRDEF_PRECISION=32' in flags + # Original default of this precision + assert '-DR_SOLVER_PRECISION=32' in flags + # Original default of this precision + assert '-DR_TRAN_PRECISION=64' in flags + # From environment variable + assert '-DR_BL_PRECISION=64' in flags + + +def test_precision_definition_with_default(monkeypatch) -> None: + ''' + Tests specification of precision. Test all ways a precision + can be specified: default command line, explicit command + line, environment variable, and the per R_*PRECISION default. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py", + "--precision-default", "32", + "--rdef_precision", "64"]) + monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) + + lfric_base = LFRicBase(name="test") + lfric_base.define_preprocessor_flags_step() + + flags = lfric_base.preprocess_flags_common + # Explicitly set on command line: + assert '-DRDEF_PRECISION=64' in flags + # Specified default of any precision + assert '-DR_SOLVER_PRECISION=32' in flags + # Specified default of any precision + assert '-DR_TRAN_PRECISION=32' in flags + # From environment variable + assert '-DR_BL_PRECISION=64' in flags + + +@pytest.mark.parametrize('no_xios', [True, False]) +@pytest.mark.parametrize('mpi', [True, False]) +def test_preprocessor_flags(monkeypatch, no_xios, mpi) -> None: + """ + Tests setting of preprocessor flags, and also that we get + the expected defaults for the precision variables. + """ + argv = ["fab_script", "--no-openmp"] + if no_xios: + argv.append("--no-xios") + if not mpi: + argv.append("--no-mpi") + monkeypatch.setattr(sys, "argv", argv) + + # Mark the compiler to have MPI or not, depending on what is needed + tr = ToolRepository() + fc = tr.get_tool(Category.FORTRAN_COMPILER, "sfc") + monkeypatch.setattr(fc, "_mpi", mpi) + + lfric_base = LFRicBase(name="test") + lfric_base.define_preprocessor_flags_step() + + expected_flags = [ + '-DRDEF_PRECISION=64', + '-DR_SOLVER_PRECISION=32', + '-DR_TRAN_PRECISION=64', + '-DR_BL_PRECISION=64' + ] + if not no_xios: + expected_flags.append("-DUSE_XIOS") + if not mpi: + expected_flags.append("-DNO_MPI") + assert set(lfric_base.preprocess_flags_common) == set(expected_flags) + + +def test_setup_site_specific_location(monkeypatch) -> None: + ''' + Tests site specific path setup for LFRicBase. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + lfric_base = LFRicBase(name="test") + + old_path = sys.path.copy() + lfric_base.setup_site_specific_location() + + # Check paths added correctly + base_dir = Path(inspect.getfile(LFRicBase)).parent + assert str(base_dir) in sys.path + assert str(base_dir / "site_specific") in sys.path + + # Restore path + sys.path = old_path + + +def test_get_linker_flags(monkeypatch) -> None: + ''' + Tests linker flags include required libraries. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + lfric_base = LFRicBase(name="test") + flags = lfric_base.get_linker_flags() + + expected_libs = ['yaxt', 'xios', 'netcdf', 'hdf5'] + assert set(flags) == set(expected_libs) + + +def test_grab_files_step(monkeypatch) -> None: + ''' + Tests grabbing required source files + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mock objects + mock_grab = mock.MagicMock() + mock_core = Path("/mock/core") + + # Setup mocks + monkeypatch.setattr('lfric_base.grab_folder', mock_grab) + + lfric_base = LFRicBase(name="test") + monkeypatch.setattr(lfric_base, '_lfric_core_root', mock_core) + + # Call method under test + lfric_base.grab_files_step() + + # Verify grab_folder called for all required directories + expected_calls = [ + # Source directories + mock.call(lfric_base.config, + src=mock_core/'infrastructure'/'source', + dst_label=''), + mock.call(lfric_base.config, + src=mock_core/'components'/'driver'/'source', + dst_label=''), + mock.call(lfric_base.config, + src=mock_core/'components'/'inventory'/'source', + dst_label=''), + mock.call(lfric_base.config, + src=mock_core/'components'/'science'/'source', + dst_label=''), + mock.call(lfric_base.config, + src=mock_core/'components'/'lfric-xios'/'source', + dst_label=''), + # PSyclone config directory + mock.call(lfric_base.config, + src=mock_core/'etc', + dst_label='psyclone_config') + ] + + # Check both number of calls and call arguments + assert mock_grab.call_count == len(expected_calls) + mock_grab.assert_has_calls(expected_calls, any_order=True) + + +def test_find_source_files_step(monkeypatch) -> None: + ''' + Tests finding and filtering source files + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mocks + with (mock.patch('lfric_base.FabBase.find_source_files_step') as find_step, + mock.patch('lfric_base.LFRicBase.templaterator_step') as temp_step, + mock.patch('lfric_base.LFRicBase.configurator_step') as conf_step, + mock.patch('lfric_base.Exclude') as mock_exclude): + lfric_base = LFRicBase(name="test") + lfric_base.find_source_files_step() + + # Verify exclusion filter added and super called + mock_exclude.assert_called_once_with('unit-test', '/test/') + find_step.assert_called_once() + # Verify configurator and templaterator called + conf_step.assert_called_once() + temp_step.assert_called_once_with(lfric_base.config) + + +def test_configurator_step(monkeypatch) -> None: + ''' + Tests the configurator setup and execution. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mock objects + mock_config = mock.MagicMock() + mock_picker = mock.MagicMock(return_value="rose_picker_tool") + mock_meta = mock.MagicMock(return_value="rose_meta.conf") + + # Set up mocks using monkeypatch + monkeypatch.setattr('lfric_base.configurator', mock_config) + monkeypatch.setattr('lfric_base.get_rose_picker', mock_picker) + + lfric_base = LFRicBase(name="test") + monkeypatch.setattr(lfric_base, 'get_rose_meta', mock_meta) + + lfric_base.configurator_step() + + # Verify configurator called with correct arguments + mock_config.assert_called_once_with( + lfric_base.config, + lfric_core_source=lfric_base.lfric_core_root, + rose_meta_conf="rose_meta.conf", + include_paths=[], + rose_picker="rose_picker_tool" + ) + + +def test_templaterator_step(monkeypatch, tmp_path) -> None: + ''' + Tests the templaterator step processes template files correctly. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mock template file + template_file = tmp_path / "field.t90" + template_file.write_text("template content", encoding='utf-8') + + # Create mock templaterator + mock_templaterator = mock.MagicMock() + mock_templaterator_instance = mock.MagicMock() + mock_templaterator.return_value = mock_templaterator_instance + monkeypatch.setattr('lfric_base.Templaterator', mock_templaterator) + + # Mock input_to_output_fpath + mock_output_path = tmp_path / "build" / "output" + mock_output_path.mkdir(parents=True) + monkeypatch.setattr('lfric_base.input_to_output_fpath', + lambda config, input_path: (mock_output_path / + input_path.name)) + + # Mock SuffixFilter to return our template file + mock_filter = mock.MagicMock() + mock_filter.return_value = {template_file} + monkeypatch.setattr('lfric_base.SuffixFilter', lambda *args: mock_filter) + + # Create mock config with proper artefact store + mock_artefact_store = mock.MagicMock() + mock_artefact_store.__getitem__.return_value = set() + + config = mock.MagicMock() + config.artefact_store = mock_artefact_store + config.build_output = tmp_path + + # Create LFRicBase instance + lfric_base = LFRicBase(name="test") + monkeypatch.setattr(lfric_base, '_lfric_core_root', tmp_path) + + # Run templaterator step + lfric_base.templaterator_step(config) + + # Verify templaterator initialization + mock_templaterator.assert_called_once_with(tmp_path / "infrastructure" / + "build" / "tools" / + "Templaterator") + + # Verify template processing + expected_calls = [] + templates = [ + {"kind": "real32", "type": "real"}, + {"kind": "real64", "type": "real"}, + {"kind": "int32", "type": "integer"} + ] + + for template in templates: + out_file = mock_output_path / f"field_{template['kind']}_mod.f90" + expected_calls.append( + mock.call(template_file, out_file, key_values=template) + ) + + assert mock_templaterator_instance.process.call_count == 3 + mock_templaterator_instance.process.assert_has_calls(expected_calls) + + # Verify artefact store add calls + expected_add_calls = [] + for template in templates: + out_file = mock_output_path / f"field_{template['kind']}_mod.f90" + expected_add_calls.append( + mock.call(ArtefactSet.FORTRAN_COMPILER_FILES, out_file) + ) + + assert mock_artefact_store.add.call_count == 3 + mock_artefact_store.add.assert_has_calls(expected_add_calls) + + # Test empty template files case + mock_filter.return_value = set() + lfric_base.templaterator_step(config) + # Call count should remain the same since no new files processed + assert mock_templaterator_instance.process.call_count == 3 + + +def test_get_rose_meta(monkeypatch) -> None: + ''' + Tests getting rose meta configuration + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + lfric_base = LFRicBase(name="test") + assert lfric_base.get_rose_meta() is None + + +def test_analyse_step(monkeypatch) -> None: + '''Tests analysis step configuration and execution''' + + # Test case 1: No ignore_dependencies argument specified + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mocks + mock_analyse = mock.MagicMock() + mock_preprocess = mock.MagicMock() + mock_psyclone = mock.MagicMock() + + # Setup mocks + monkeypatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', + mock_analyse) + + lfric_base = LFRicBase(name="test") + + # Mock instance methods + monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) + monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) + + # Call analyse_step + lfric_base.analyse_step() + + # Verify method calls + mock_preprocess.assert_called_once() + mock_psyclone.assert_called_once() + + # Verify analyse called with correct default ignore_dependencies + expected_ignore = ['netcdf', 'mpi', 'mpi_f08', 'yaxt', + 'xios', 'icontext', 'mod_wait'] + mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) + + # Test case 2: Custom ignore_dependencies arguments specified + custom_ignore = ['custom_dep1', 'custom_dep2'] + mock_analyse.reset_mock() + mock_preprocess.reset_mock() + mock_psyclone.reset_mock() + + lfric_base = LFRicBase(name="test") + monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) + monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) + + # Call analyse_step + lfric_base.analyse_step(ignore_dependencies=custom_ignore) + + # Verify methods still called + mock_preprocess.assert_called_once() + mock_psyclone.assert_called_once() + + # Verify analyse called with custom_ignore added to ignore list + expected_ignore = ['custom_dep1', 'custom_dep2', 'netcdf', 'mpi', + 'mpi_f08', 'yaxt', 'xios', 'icontext', + 'mod_wait'] + mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) + + +def test_preprocess_x90_step(monkeypatch) -> None: + ''' + Tests preprocessing of X90 files. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + mock_preproc = mock.MagicMock() + monkeypatch.setattr('lfric_base.preprocess_x90', mock_preproc) + + lfric_base = LFRicBase(name="test") + lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) + lfric_base.preprocess_x90_step() + + mock_preproc.assert_called_once_with( + lfric_base.config, + common_flags=["-flag1", "-flag2"] + ) + + +def test_psyclone_step(monkeypatch) -> None: + ''' + Tests the PSyclone step. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create mock objects + mock_psy = mock.MagicMock() + mock_config_opts = ["--config", "/mock/psyclone.cfg"] + mock_additional_opts: List[str] = [] + + # Set up monkeypatch for module level import + monkeypatch.setattr('lfric_base.psyclone', mock_psy) + + lfric_base = LFRicBase(name="test") + + # Patch instance methods. Return a copy to avoid that + # PSyclone modified these lists in the lambdas when it modifies the list + monkeypatch.setattr(lfric_base, 'get_psyclone_config', + lambda: mock_config_opts[:]) + monkeypatch.setattr(lfric_base, 'get_additional_psyclone_options', + lambda: mock_additional_opts[:]) + + # Call method under test + lfric_base.psyclone_step(additional_parameters=["-additional"]) + + # Verify psyclone called with correct arguments + print(mock_psy.mock_calls) + print("UUU", mock_config_opts, mock_additional_opts) + mock_psy.assert_called_once_with( + lfric_base.config, + kernel_roots=[(lfric_base.config.build_output / "kernel")], + transformation_script=lfric_base.get_transformation_script, + api="dynamo0.3", + cli_args=mock_config_opts + mock_additional_opts + ["-additional"], + ignore_dependencies=None + ) + + +def test_get_psyclone_config(monkeypatch) -> None: + ''' + Tests getting PSyclone config. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + lfric_base = LFRicBase(name="test") + config_args = lfric_base.get_psyclone_config() + + assert config_args == ["--config", + str(lfric_base.config.source_root / + 'psyclone_config/psyclone.cfg')] + + +def test_get_additional_psyclone_options(monkeypatch) -> None: + ''' + Tests getting additional PSyclone options (for profiling). + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + lfric_base = LFRicBase(name="test") + assert not lfric_base.get_additional_psyclone_options() + + +def test_get_transformation_script(monkeypatch, tmp_path) -> None: + ''' + Tests finding PSyclone transformation scripts. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + + # Create LFRicBase instance with mocked site/platform + lfric_base = LFRicBase(name="test") + + # Create mock config + config = mock.MagicMock() + config.source_root = tmp_path + config.build_output = tmp_path / "build" + config.build_output.mkdir() + + # Create x90 test source file + source_path = tmp_path / "some/path" + source_path.mkdir(parents=True) + test_file = source_path / "file.x90" + test_file.touch() + + # Test case 1: x90 file not in source or build directories + outside_file = tmp_path.parent / "outside.x90" + assert lfric_base.get_transformation_script(outside_file, config) is None + + # Test case 2: No optimisation directory, no transformation script + assert lfric_base.get_transformation_script(test_file, config) is None + + # Test case 3: No PSykal but optimisation directory + optimisation_folder_path = (tmp_path / "optimisation" / "default-default" / + "psykal") + global_script = optimisation_folder_path / "global.py" + global_script.parent.mkdir(parents=True) + global_script.touch() + + # No file-specific transformation script, use global script + other_file = tmp_path / "other/path/test.x90" + other_file.parent.mkdir(parents=True) + other_file.touch() + assert (lfric_base.get_transformation_script(other_file, config) == + global_script) + + # Test case 4: Psykal directory exists + psykal_path = tmp_path / "optimisation/default-default/psykal" + + # Create specific transformation script in psykal dir + specific_script = psykal_path / "some/path/file.py" + specific_script.parent.mkdir(parents=True) + specific_script.touch() + + # Use specific script in psykal directory + assert lfric_base.get_transformation_script(test_file, config) == \ + specific_script diff --git a/infrastructure/build/fab/test/rose_picker_tool_test.py b/infrastructure/build/fab/test/rose_picker_tool_test.py new file mode 100644 index 000000000..f0993174c --- /dev/null +++ b/infrastructure/build/fab/test/rose_picker_tool_test.py @@ -0,0 +1,125 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author J. Henrichs, Bureau of Meteorology + +""" +This module tests rose_picker_tool. +""" + +import os +from pathlib import Path +from unittest.mock import patch, MagicMock, PropertyMock + +import pytest + +from fab.tools.category import Category +from fab.tools.tool import Tool +from rose_picker_tool import get_rose_picker, RosePicker + + +def test_get_rose_picker_system_found() -> None: + """ + Test that a system-wide installed rose_picker works as expected. + """ + with patch("shutil.which", return_value="/usr/bin/rose_picker"): + rp = get_rose_picker("system") + assert isinstance(rp, RosePicker) + assert rp.exec_path == Path("/usr/bin/rose_picker") + + +def test_get_rose_picker_system_not_found() -> None: + """ + Test error if a system rose_picker is requested, but does not exist. + """ + with patch("shutil.which", return_value=None): + with pytest.raises(RuntimeError) as err: + get_rose_picker("system") + assert "Cannot find system rose_picker tool." == str(err.value) + + +def test_get_rose_picker_local_checkout(tmp_path) -> None: + """ + Tests that we will invoke rose picker from a local checkout + (mocked, so we don't need an actual checkout) + """ + tag = "v2.0.0" + fake_workspace = tmp_path / "fab-workspace" + gpl_utils = fake_workspace / f"gpl-utils-{tag}" / "source" + rose_picker_bin = gpl_utils / "bin" + rose_picker_path = rose_picker_bin / "rose_picker" + + # Patch get_fab_workspace to return our tmp_path + pm = PropertyMock("is_available", side_effect=[False, True]) + with patch("rose_picker_tool.get_fab_workspace", + return_value=fake_workspace), \ + patch("rose_picker_tool.ToolRepository") as mock_repo_class, \ + patch.object(Tool, "is_available", pm): + + mock_fcm = MagicMock() + mock_repo = MagicMock() + mock_repo.get_default.return_value = mock_fcm + mock_repo_class.return_value = mock_repo + + rp = get_rose_picker(tag) + + # Ensure checkout was called + mock_fcm.checkout.assert_called_once_with( + src=f"fcm:lfric_gpl_utils.x/tags/{tag}", + dst=gpl_utils + ) + + # Ensure the returned object is a RosePicker with correct path + assert isinstance(rp, RosePicker) + assert Path(rp.exec_path) == rose_picker_path + + +def test_get_rose_picker_local_checkout_fails() -> None: + """ + This functions tests the behaviour if a local checkout fails, + i.e. rose_picker cannot be executed. This test patches the + ToolRepository (so that FCM is not actually called), and makes + sure RosePicker is always not available: + """ + + tag = "v2.0.0" + + # Make sure rose_picker will always return to be not available: + with patch("rose_picker_tool.ToolRepository.get_default") as mock_repo, \ + patch.object(RosePicker, "check_available", return_value=False), \ + pytest.raises(RuntimeError) as err: + get_rose_picker(tag) + + assert f"Cannot run rose_picker tag '{tag}'." == str(err.value) + # Also make sure that we indeed got FCM :) + mock_repo.assert_called_with(Category.FCM) + + +def test_get_rose_picker_check_available() -> None: + """ + Test RosePicker's check_available. + """ + rose_picker = RosePicker(Path("/usr/bin/rose_picker")) + with patch.object(RosePicker, "run", return_value=True) as mock_run: + assert rose_picker.check_available() + mock_run.assert_called_once_with(additional_parameters="-help") + + with patch.object(RosePicker, "run", side_effect=RuntimeError) as mock_run: + assert not rose_picker.check_available() + mock_run.assert_called_once_with(additional_parameters="-help") + + +def test_get_rose_picker_execute() -> None: + """ + Test RosePicker's check_available. + """ + rose_picker = RosePicker(Path("/usr/bin/rose_picker")) + with patch.object(RosePicker, "run", return_value=0) as mock_run, \ + patch.object(os, "environ", {}): + rose_picker.execute(["arg"]) + # Rose picker prepends the existing python path, separated by ":". + # Since python path is not set, there will be a leading ":"" + mock_run.assert_called_once_with(additional_parameters=["arg"], + env={'PYTHONPATH': ':/usr/lib/python'}) diff --git a/infrastructure/build/fab/test/templaterator_test.py b/infrastructure/build/fab/test/templaterator_test.py new file mode 100644 index 000000000..e50f3f62a --- /dev/null +++ b/infrastructure/build/fab/test/templaterator_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Henrichs, Bureau of Meteorology + +""" +This module tests the Templaterator tool. +""" + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from templaterator import Templaterator # adjust import as needed + + +@pytest.fixture(name="templaterator") +def templaterator_setup(tmp_path: Path) -> Path: + """ + :returns: A dummy Templaterator object (in tmp_path), + which does not exist). + """ + return Templaterator(tmp_path / "Templaterator.py") + + +def test_init(templaterator: Templaterator, + tmp_path: Path) -> None: + """ + Test the constructor. + """ + assert templaterator.name == "Templaterator" + assert templaterator.exec_path == tmp_path / "Templaterator.py" + assert templaterator.exec_name == "Templaterator.py" + + +def test_check_available(templaterator: Templaterator) -> None: + """ + Test the check_available function. + """ + with patch("fab.tools.tool.Tool.run", return_value=0) as mock_run: + assert templaterator.check_available() is True + mock_run.assert_called_once_with(additional_parameters="-h") + + with patch("fab.tools.tool.Tool.run", + side_effect=RuntimeError()) as mock_run: + assert templaterator.check_available() is False + mock_run.assert_called_once_with(additional_parameters="-h") + + +def test_process_call(templaterator: Templaterator, + tmp_path: Path) -> None: + """ + Test that execution passes on the right parameter to the + Templaterator script. + """ + input_template = tmp_path / "input.txt" + output_file = tmp_path / "output.txt" + key_values = {"A": "1", "B": "2"} + + with patch("fab.tools.tool.Tool.run", return_value=0) as mock_run: + templaterator.process(input_template, output_file, key_values) + + expected_params = [ + input_template, + "-o", + output_file, + "-s A=1", + "-s B=2", + ] + + mock_run.assert_called_once_with(additional_parameters=expected_params) From 962ce0a6f86a331c54087117179764991ecf0923 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 12:54:38 +1100 Subject: [PATCH 05/95] #240 Updated skeleton build script. --- applications/skeleton/fab_skeleton.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index 8bd4d6e1e..11df842c1 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 -# ############################################################################## -# (c) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT -# which you should have received as part of this distribution -# ############################################################################## -'''A FAB build script for applications/skeleton. It relies on +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Henrichs, Bureau of Meteorology +# Author: J. Lyu, Bureau of Meteorology + +""" +A FAB build script for applications/skeleton. It relies on the LFRicBase class contained in the infrastructure directory. -''' +""" import logging from pathlib import Path From 54491da79784a06d215d2ed76f29df308ef5d772 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 13:06:49 +1100 Subject: [PATCH 06/95] #240 Fixed incorrect typing in test. --- infrastructure/build/fab/test/templaterator_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/infrastructure/build/fab/test/templaterator_test.py b/infrastructure/build/fab/test/templaterator_test.py index e50f3f62a..c65e3ba95 100644 --- a/infrastructure/build/fab/test/templaterator_test.py +++ b/infrastructure/build/fab/test/templaterator_test.py @@ -20,10 +20,11 @@ @pytest.fixture(name="templaterator") -def templaterator_setup(tmp_path: Path) -> Path: +def templaterator_setup(tmp_path: Path) -> Templaterator: """ - :returns: A dummy Templaterator object (in tmp_path), - which does not exist). + :returns: A dummy Templaterator object (in tmp_path, + which does not exist, but it's all we need for these tests since + all tests are mocked. """ return Templaterator(tmp_path / "Templaterator.py") From 22c2ef930327686826dc26e287af92ade2e0d425 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 14:38:36 +1100 Subject: [PATCH 07/95] #240 Added README file. --- infrastructure/build/fab/README.md | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 infrastructure/build/fab/README.md diff --git a/infrastructure/build/fab/README.md b/infrastructure/build/fab/README.md new file mode 100644 index 000000000..882780351 --- /dev/null +++ b/infrastructure/build/fab/README.md @@ -0,0 +1,63 @@ +# LFRic Core Fab Build Scripts + +Make sure you have Fab version 2.0.1 or later installed (in addition to all +LFRic core requirements of course). + +## Setting up Site- and Platform-specific Settings +Site- and platform-specific settings are contained in +```$LFRIC_CORE/infrastructure/build/site-specific/${SITE}-${PLATFORM}``` +The default settings are in ```.../site-specific/default``` (and at this +stage each other site-specific setup inherits the values set in the default, +and then adds or modifies settings). The Fab build system provides various +callbacks to the ```config.py``` file in the corresponding directory (details +are in the [Fab documentation](https://metoffice.github.io/fab/fab_base/config.html). + +If there is no existing site-specific setup, it is recommended to copy an existing +configuration file (e.g. from ```nci_gadi/config.py```). This an act as a template +to indicate where you can specify linker information, select a default compiler +suite etc. + +The default setup contains compiler flags for Cray, GNU, Intel-classic (ifort), +Intel-LLVM (ifx), and NVIDIA. For modularity's sake (and to keep the file length +shorter), the default configuration will get the settings from the corresponding +```setup_...py``` script. There is no need for a site to replicate this structure, +existing ```config.py``` scripts show how this can be done. + + +## Setting up PYTHONPATH +Many (if not all) PSyclone scripts use a library of convenience functions located in +```$LFRIC_CORE/infrastructure/build/psyclone/psyclone_tools.py```. In order to be +able to import these functions, the path ```$LFRIC_CORE/infrastructure/build/psyclone``` +must be added to your ```$PYTHONPATH```, e.g.: +``` +export PYTHONPATH=$LFRIC_CORE/infrastructure/build/psyclone:$PYTHONPATH +``` + +## Building the Skeleton Apps + +In order to build the skeleton apps, change into the directory +```$LFRIC_CORE/applications/skeleton```, +and use the following command: + +``` +./fab_skeleton.py --nprocs 4 --site nci --platform gadi --suite intel-classic +``` +Select an appropriate number of processes to run in parallel, and your site and platform. +If you don't have specified a default compiler suite in your site-specific setup (or +want to use a non-default suite), use the ``--suite`` option. Once the process is finished, +you should have a binary in the directory +```./fab-workspace/skeleton-full-debug-COMPILER``` (where ```COMPILER``` is the compiler +used, e.g. ```mpif90-gfortran```). + +Using ```./fab_skeleton.py -h``` will show a help message with all supported command line +options (and their default value). If a default value is listed using an environment +variables (```(default: $SITE or 'default')```), the corresponding environment variable +is used if no command line option has been specified. + +A different compilation profile can be specified using ```--profile``` option. Note +that the available compilation profiles can vary from site to site (see +[Fab documentation](https://metoffice.github.io/fab/fab_base/config.html) for details). + +If Fab has issues finding a compiler, you can use the Fab debug option +```--available-compilers```, which will list all compilers and linkers Fab has +identified as being available. From 25c26fb665ad2a359d6295f95e9d61014c3d0872 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 14:40:31 +1100 Subject: [PATCH 08/95] #240 Removed fab and vernier support from NCI site. --- .../fab/site_specific/nci_gadi/config.py | 74 +------------------ 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/infrastructure/build/fab/site_specific/nci_gadi/config.py b/infrastructure/build/fab/site_specific/nci_gadi/config.py index cf826774f..fd9a18972 100644 --- a/infrastructure/build/fab/site_specific/nci_gadi/config.py +++ b/infrastructure/build/fab/site_specific/nci_gadi/config.py @@ -10,69 +10,11 @@ from pathlib import Path from typing import List, Union, Optional -from fab.api import (BuildConfig, Category, Compiler, CompilerWrapper, - ToolRepository) +from fab.api import BuildConfig, Category, Compiler, ToolRepository from default.config import Config as DefaultConfig -class Tauf90(CompilerWrapper): - ''' - Class for the Tau profiling Fortran compiler wrapper. - It will be using the name "tau-COMPILER_NAME", but will call tau_f90.sh. - - :param compiler: the compiler that the tau_f90.sh wrapper will use. - :type compiler: :py:class:`fab.tools.Compiler` - ''' - - def __init__(self, compiler: Compiler): - super().__init__(name=f"tau-{compiler.name}", - exec_name="tau_f90.sh", compiler=compiler, mpi=True) - - def compile_file(self, input_file: Path, - output_file: Path, - config: BuildConfig, - add_flags: Union[None, List[str]] = None, - syntax_only: Optional[bool] = None) -> None: - ''' - This method overrides the Fab CompilerWrapper class compile_file - method to fall back to the wrapped compiler for certain Fortran files - and use the tau_f90.sh wrapper to compile the rest. - - :param Path input_file: the path of the input file to compile - :param Path output_file: the path of the output file to create - :param config: the Fab build configuration instance - :type config: :py:class:`fab.BuildConfig` - :param add_flags: additional flags to pass to the compiler - :type add_flags: Union[None, List[str]] - :param syntax_only: whether to only check the syntax of the file - :type syntax_only: Optional[bool] - ''' - if ('psy.f90' in str(input_file)) or \ - ('/kernel/' in str(input_file)) or \ - ('leaf_jls_mod' in str(input_file)) or \ - ('/science/' in str(input_file)): - self.compiler.compile_file(input_file, output_file, - config, add_flags, syntax_only) - else: - super().compile_file(input_file, output_file, - config, add_flags, syntax_only) - - -class Taucc(CompilerWrapper): - ''' - Class for the Tau profiling C compiler wrapper. - It will be using the name "tau-COMPILER_NAME", but will call tau_cc.sh. - - :param compiler: the compiler that the tau_cc.sh wrapper will use - :type compiler: :py:class:`fab.tools.Compiler` - ''' - - def __init__(self, compiler: Compiler): - super().__init__(name=f"tau-{compiler.name}", - exec_name="tau_cc.sh", compiler=compiler, mpi=True) - - class Config(DefaultConfig): ''' For NCI, make intel the default, and add the Tau wrapper. @@ -83,16 +25,6 @@ def __init__(self): tr = ToolRepository() tr.set_default_compiler_suite("intel-classic") - # Add the tau wrappers for Fortran and C. Note that add_tool - # will automatically add them as a linker as well. - for ftn in ["ifort", "gfortran"]: - compiler = tr.get_tool(Category.FORTRAN_COMPILER, ftn) - tr.add_tool(Tauf90(compiler)) - - for cc in ["icc", "gcc"]: - compiler = tr.get_tool(Category.C_COMPILER, cc) - tr.add_tool(Taucc(compiler)) - # ATM we don't use a shell when running a tool, and as such # we can't directly use "$()" as parameter. So query these values using # Fab's shell tool (doesn't really matter which shell we get, so just @@ -102,13 +34,13 @@ def __init__(self): nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], capture_output=True).strip().split() linker = tr.get_tool(Category.LINKER, "linker-tau-ifort") + + # Setup all linker flags: linker.add_lib_flags("netcdf", nc_flibs) linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) linker.add_lib_flags("shumlib", ["-lshum"]) - linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", - "-lvernier"]) # Always link with C++ libs linker.add_post_lib_flags(["-lstdc++"]) From 72192df236683f54a3b787096cb0837fc5373894 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 16:30:09 +1100 Subject: [PATCH 09/95] #240 Fixed typos. --- infrastructure/build/fab/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/build/fab/README.md b/infrastructure/build/fab/README.md index 882780351..dc1e2bfcd 100644 --- a/infrastructure/build/fab/README.md +++ b/infrastructure/build/fab/README.md @@ -13,7 +13,7 @@ callbacks to the ```config.py``` file in the corresponding directory (details are in the [Fab documentation](https://metoffice.github.io/fab/fab_base/config.html). If there is no existing site-specific setup, it is recommended to copy an existing -configuration file (e.g. from ```nci_gadi/config.py```). This an act as a template +configuration file (e.g. from ```nci_gadi/config.py```). This act as a template to indicate where you can specify linker information, select a default compiler suite etc. @@ -43,7 +43,7 @@ and use the following command: ./fab_skeleton.py --nprocs 4 --site nci --platform gadi --suite intel-classic ``` Select an appropriate number of processes to run in parallel, and your site and platform. -If you don't have specified a default compiler suite in your site-specific setup (or +If you don't have a default compiler suite in your site-specific setup (or want to use a non-default suite), use the ``--suite`` option. Once the process is finished, you should have a binary in the directory ```./fab-workspace/skeleton-full-debug-COMPILER``` (where ```COMPILER``` is the compiler From f7a839c4fad93bcdcbc099af9eb4b89d30a0f113 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 16:45:49 +1100 Subject: [PATCH 10/95] #240 Allow import of psyclone_tools without setting PYTHONPATH. --- infrastructure/build/fab/README.md | 9 --------- infrastructure/build/fab/lfric_base.py | 8 ++++++++ infrastructure/build/fab/test/lfric_base_test.py | 6 +++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/infrastructure/build/fab/README.md b/infrastructure/build/fab/README.md index dc1e2bfcd..5c22299be 100644 --- a/infrastructure/build/fab/README.md +++ b/infrastructure/build/fab/README.md @@ -24,15 +24,6 @@ shorter), the default configuration will get the settings from the corresponding existing ```config.py``` scripts show how this can be done. -## Setting up PYTHONPATH -Many (if not all) PSyclone scripts use a library of convenience functions located in -```$LFRIC_CORE/infrastructure/build/psyclone/psyclone_tools.py```. In order to be -able to import these functions, the path ```$LFRIC_CORE/infrastructure/build/psyclone``` -must be added to your ```$PYTHONPATH```, e.g.: -``` -export PYTHONPATH=$LFRIC_CORE/infrastructure/build/psyclone:$PYTHONPATH -``` - ## Building the Skeleton Apps In order to build the skeleton apps, change into the directory diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 57744132b..a438f8f14 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -64,6 +64,10 @@ def __init__(self, name: str, self._psyclone_config = (self.config.source_root / 'psyclone_config' / 'psyclone.cfg') + # Many PSyclone scripts use module(s) from this directory. Additional + # paths might need to be added later. + self._add_python_paths = [str(self.lfric_core_root / "infrastructure" / + "build" / "psyclone")] def define_command_line_options( self, @@ -374,12 +378,16 @@ def psyclone_step( if additional_parameters: psyclone_cli_args.extend(additional_parameters) + # To avoid impacting other code, store the original search path + old_sys_path = sys.path[:] + sys.path.extend(self._add_python_paths) psyclone(self.config, kernel_roots=[(self.config.build_output / "kernel")], transformation_script=self.get_transformation_script, api="dynamo0.3", cli_args=psyclone_cli_args, ignore_dependencies=ignore_dependencies) + sys.path = old_sys_path def get_psyclone_config(self) -> List[str]: ''' diff --git a/infrastructure/build/fab/test/lfric_base_test.py b/infrastructure/build/fab/test/lfric_base_test.py index c07279282..9ae1daef9 100644 --- a/infrastructure/build/fab/test/lfric_base_test.py +++ b/infrastructure/build/fab/test/lfric_base_test.py @@ -573,8 +573,12 @@ def test_analyse_step(monkeypatch) -> None: monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) - # Call analyse_step + # The PSyclone step will modify sys.path (to allow import of + # psyclone_tools by PSyclone scripts). Make sure sys.path is unchanged: + old_sys_path = sys.path[:] + # Call analyse_step (which calls PSyclone) lfric_base.analyse_step() + assert sys.path == old_sys_path # Verify method calls mock_preprocess.assert_called_once() From 170cab3243c041622090dcf126a2718128bdd023 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 16:46:47 +1100 Subject: [PATCH 11/95] #240 Fixed flake8 errors. --- infrastructure/build/fab/site_specific/nci_gadi/config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/infrastructure/build/fab/site_specific/nci_gadi/config.py b/infrastructure/build/fab/site_specific/nci_gadi/config.py index fd9a18972..20d015b9a 100644 --- a/infrastructure/build/fab/site_specific/nci_gadi/config.py +++ b/infrastructure/build/fab/site_specific/nci_gadi/config.py @@ -7,10 +7,7 @@ - Adds the tau compiler wrapper as (optional) compilers to the ToolRepository. ''' -from pathlib import Path -from typing import List, Union, Optional - -from fab.api import BuildConfig, Category, Compiler, ToolRepository +from fab.api import Category, ToolRepository from default.config import Config as DefaultConfig From d7aa9f1363a88a4138e307caddef84faa9e68d11 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 27 Jan 2026 17:13:31 +1100 Subject: [PATCH 12/95] #240 Added myself to contributors list. --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d0f7ae14d..40285672a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,3 +4,4 @@ | ----------- | --------- | ----------- | ---- | | james-bruten-mo | James Bruten | Met Office | 2025-12-09 | | jennyhickson | Jenny Hickson | Met Office | 2025-12-10 | +| hiker | Joerg Henrichs | Bureau of Meteorology | 2026-01-22 | From 9ee0b647eb58ffa4280ac9d937346ee743c3ac6b Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sun, 8 Feb 2026 11:25:11 +1100 Subject: [PATCH 13/95] Fixed rose picker etc issues raised in review. --- infrastructure/build/fab/configurator.py | 10 +- infrastructure/build/fab/lfric_base.py | 19 +-- infrastructure/build/fab/rose_picker_tool.py | 112 ++++-------------- .../build/fab/test/configurator_test.py | 27 ++--- .../build/fab/test/rose_picker_tool_test.py | 109 +++-------------- 5 files changed, 55 insertions(+), 222 deletions(-) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index 1c40f383a..fdea9a75a 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -25,7 +25,6 @@ def configurator(config: BuildConfig, lfric_core_source: Path, rose_meta_conf: Path, - rose_picker: RosePicker, include_paths: Optional[list[Path]] = None, config_dir: Optional[Path] = None) -> None: """ @@ -34,7 +33,6 @@ def configurator(config: BuildConfig, :param config: the Fab build config instance :param lfric_core_source: the path to the LFRic core directory :param rose_meta_conf: the path to the rose-meta configuration file - :param rose_picker: the rose picker tool :param include_paths: additional include paths (each path will be added, as well as the path with /'rose-meta') :param config_dir: the directory for the generated configuration files @@ -55,11 +53,9 @@ def configurator(config: BuildConfig, for path in include_paths: include_dirs.extend([path, path / 'rose-meta']) - parameters = [rose_meta_conf, '-directory', config_dir] - for incl_dir in include_dirs: - parameters.extend(['-include_dirs', incl_dir]) - - rose_picker.execute(parameters=parameters) + rose_picker = RosePicker() + rose_picker.execute(rose_meta_conf, config_dir, + include_paths=include_dirs) rose_meta = config_dir / 'rose-meta.json' shell = config.tool_box.get_tool(Category.SHELL) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index a438f8f14..0b83cb097 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -24,7 +24,6 @@ from fab.fab_base.fab_base import FabBase from configurator import configurator -from rose_picker_tool import get_rose_picker from templaterator import Templaterator @@ -75,8 +74,8 @@ def define_command_line_options( ) -> argparse.ArgumentParser: ''' This adds LFRic specific command line options to the base class - define_command_line_option. Currently, --rose_picker and - precision-related options are added. + define_command_line_option. Currently, precision-related options + are added. :param parser: optional a pre-defined argument parser. @@ -84,11 +83,6 @@ def define_command_line_options( ''' parser = super().define_command_line_options() - parser.add_argument( - '--rose_picker', '-rp', type=str, default="system", - help="Version of rose_picker. Use 'system' to use an installed " - "version.") - parser.add_argument( '--no-xios', action="store_true", default=False, help="Disable compilation with XIOS.") @@ -265,12 +259,6 @@ def configurator_step( ''' rose_meta = self.get_rose_meta() if rose_meta: - # Get the right version of rose-picker, depending on - # command line option (defaulting to v2.0.0) - # TODO: Ideally we would just put this into the toolbox, - # but atm we can't put several tools of one category in - # (so ToolBox will need to support more than one MISC tool) - rp = get_rose_picker(self.args.rose_picker) # Ideally we would want to get all source files created in # the build directory, but then we need to know the list of # files to add them to the list of files to process. Instead, @@ -279,8 +267,7 @@ def configurator_step( include_paths = include_paths or [] configurator(self.config, lfric_core_source=self.lfric_core_root, rose_meta_conf=rose_meta, - include_paths=include_paths, - rose_picker=rp) + include_paths=include_paths) def templaterator_step(self, config: BuildConfig) -> None: ''' diff --git a/infrastructure/build/fab/rose_picker_tool.py b/infrastructure/build/fab/rose_picker_tool.py index 202af1c07..d3649bac9 100755 --- a/infrastructure/build/fab/rose_picker_tool.py +++ b/infrastructure/build/fab/rose_picker_tool.py @@ -14,107 +14,35 @@ """ import logging -import os from pathlib import Path -import shutil -from typing import cast, List, Union -from fab.api import Category, Tool, ToolRepository -from fab.tools.versioning import Fcm -from fab.util import get_fab_workspace +from fab.api import Tool -logger = logging.getLogger('fab') +logger = logging.getLogger(__name__) class RosePicker(Tool): - '''This implements rose_picker as a Fab tool. It supports dynamically - adding the required PYTHONPATH to the environment in case that rose_picker - is not installed, but downloaded. + '''This implements rose_picker as a Fab tool. - :param Path path: the path to the rose picker binary. + :param path: the path to the rose picker binary. ''' - def __init__(self, path: Path): - super().__init__("rose_picker", exec_name=str(path)) - # This is the required PYTHONPATH for running rose_picker - # when it is installed from the repository: - self._pythonpath = path.parents[1] / "lib" / "python" - - def check_available(self) -> bool: - ''' - :returns bool: whether rose_picker works by running - `rose_picker -help`. - ''' - try: - self.run(additional_parameters="-help") - except RuntimeError: - return False - - return True - - def execute(self, parameters: List[Union[Path, str]]) -> None: + def __init__(self): + super().__init__("rose_picker", exec_name="rose_picker", + availability_option="-help") + + def execute(self, + rose_meta_conf: Path, + directory: Path, + include_paths: list[Path]) -> None: ''' - This wrapper adds the required PYTHONPATH, and passes all - parameters through to the tool's run function. - :param additional_parameter: A list of parameters for rose picker. + :param rose_meta_conf: Path to the metadata file to load. + :param directory: Path to the output directory. + :param include_paths: List of include directories which are + searched for inherited metadata files. ''' - env = os.environ.copy() - env["PYTHONPATH"] = (f"{env.get('PYTHONPATH', '')}:" - f"{self._pythonpath}") - - self.run(additional_parameters=parameters, env=env) - - -# ============================================================================= -def get_rose_picker(tag: str = "v2.0.0") -> RosePicker: - ''' - Returns a Fab RosePicker tool. It can either be a version installed - in the system, which is requested by setting tag to `system`, or a - newly installed version via an FCM checkout. If there is already a - checked-out version, it will be used (i.e. no repeated downloads are - done). - - :param tag: Either the tag in the repository to use, - or 'system' to indicate to use a version installed in the system. - - :returns RosePicker: a Fab RosePicker tool instance - ''' - - if tag.lower() == "system": - # 'system' means to use a rose_picker installed in the system - which_rose_picker = shutil.which("rose_picker") - if not which_rose_picker: - raise RuntimeError("Cannot find system rose_picker tool.") - return RosePicker(Path(which_rose_picker)) - - # Otherwise use rose_picker from the default Fab workspace. It will - # create a instance of the class above, which will add its path to - # PYTHONPATH when executing a rose_picker command. - - gpl_utils = get_fab_workspace() / f"gpl-utils-{tag}" / "source" - rp_path = gpl_utils / "bin" / "rose_picker" - rp = RosePicker(rp_path) - - # If the tool is not available (the class will run `rose_picker -help` - # to verify this ), install it - if not rp.is_available: - fcm = ToolRepository().get_default(Category.FCM) - fcm = cast(Fcm, fcm) - # TODO: atm we are using fcm for the checkout, because using FCM - # keywords is more portable. We cannot use a Fab config (since this - # function is called from within a Fab build), so that means the - # gpl-utils-* directories in the Fab workspace directories do not - # have the normal directory layout. - logger.info(f"Installing rose_picker tag '{tag}'.") - fcm.checkout(src=f'fcm:lfric_gpl_utils.x/tags/{tag}', - dst=gpl_utils) - - # We need to create a new instance, since `is_available` is - # cached (I.e. it's always false in the previous instance) - rp = RosePicker(rp_path) + params = [rose_meta_conf, "-directory", directory] + for inc_path in include_paths: + params.extend(["-include_dirs", inc_path]) - if not rp.is_available: - msg = f"Cannot run rose_picker tag '{tag}'." - logger.exception(msg) - raise RuntimeError(msg) - return rp + super().run(additional_parameters=params) diff --git a/infrastructure/build/fab/test/configurator_test.py b/infrastructure/build/fab/test/configurator_test.py index 0bb3f0373..12ca84698 100644 --- a/infrastructure/build/fab/test/configurator_test.py +++ b/infrastructure/build/fab/test/configurator_test.py @@ -9,7 +9,7 @@ This module tests the configurator. """ -from unittest.mock import MagicMock +from unittest.mock import patch, MagicMock import pytest @@ -19,8 +19,8 @@ from fab.build_config import BuildConfig -@pytest.fixture -def mock_shell(): +@pytest.fixture(name="mock_shell") +def mock_shell_fixture(): """ A simple shell mock to check that all expected calls are executed. """ @@ -61,12 +61,14 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): config_namelist.write_text("namelist1\nnamelist2\n", encoding="utf8") # Run configurator - with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): + with patch("rose_picker_tool.RosePicker.execute", + return_value=0) as rose_picker, \ + pytest.warns(match="_metric_send_conn not set, cannot " + "send metrics"): configurator( config=config, lfric_core_source=lfric_core, rose_meta_conf=rose_meta_conf, - rose_picker=rose_picker, include_paths=[lfric_apps], config_dir=config_dir ) @@ -74,16 +76,11 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): tools_dir = lfric_core / "infrastructure" / "build" / "tools" # Check rose_picker was called with the expected arguments: - rose_picker.execute.assert_called_once() - kwargs = rose_picker.execute.call_args_list[0].kwargs - assert kwargs["parameters"] == [ - rose_meta_conf, - '-directory', config_dir, - '-include_dirs', lfric_core, - '-include_dirs', lfric_core / "rose-meta", - '-include_dirs', lfric_apps, - '-include_dirs', lfric_apps / "rose-meta" - ] + rose_picker.assert_called_once() + rose_picker.assert_called_with( + rose_meta_conf, config_dir, + include_paths=[lfric_core, lfric_core / "rose-meta", + lfric_apps, lfric_apps / "rose-meta"]) # Check shell.exec was called with expected commands expected_calls = [ diff --git a/infrastructure/build/fab/test/rose_picker_tool_test.py b/infrastructure/build/fab/test/rose_picker_tool_test.py index f0993174c..722ac9106 100644 --- a/infrastructure/build/fab/test/rose_picker_tool_test.py +++ b/infrastructure/build/fab/test/rose_picker_tool_test.py @@ -11,115 +11,40 @@ import os from pathlib import Path -from unittest.mock import patch, MagicMock, PropertyMock +from unittest.mock import patch -import pytest - -from fab.tools.category import Category -from fab.tools.tool import Tool -from rose_picker_tool import get_rose_picker, RosePicker - - -def test_get_rose_picker_system_found() -> None: - """ - Test that a system-wide installed rose_picker works as expected. - """ - with patch("shutil.which", return_value="/usr/bin/rose_picker"): - rp = get_rose_picker("system") - assert isinstance(rp, RosePicker) - assert rp.exec_path == Path("/usr/bin/rose_picker") - - -def test_get_rose_picker_system_not_found() -> None: - """ - Test error if a system rose_picker is requested, but does not exist. - """ - with patch("shutil.which", return_value=None): - with pytest.raises(RuntimeError) as err: - get_rose_picker("system") - assert "Cannot find system rose_picker tool." == str(err.value) - - -def test_get_rose_picker_local_checkout(tmp_path) -> None: - """ - Tests that we will invoke rose picker from a local checkout - (mocked, so we don't need an actual checkout) - """ - tag = "v2.0.0" - fake_workspace = tmp_path / "fab-workspace" - gpl_utils = fake_workspace / f"gpl-utils-{tag}" / "source" - rose_picker_bin = gpl_utils / "bin" - rose_picker_path = rose_picker_bin / "rose_picker" - - # Patch get_fab_workspace to return our tmp_path - pm = PropertyMock("is_available", side_effect=[False, True]) - with patch("rose_picker_tool.get_fab_workspace", - return_value=fake_workspace), \ - patch("rose_picker_tool.ToolRepository") as mock_repo_class, \ - patch.object(Tool, "is_available", pm): - - mock_fcm = MagicMock() - mock_repo = MagicMock() - mock_repo.get_default.return_value = mock_fcm - mock_repo_class.return_value = mock_repo - - rp = get_rose_picker(tag) - - # Ensure checkout was called - mock_fcm.checkout.assert_called_once_with( - src=f"fcm:lfric_gpl_utils.x/tags/{tag}", - dst=gpl_utils - ) - - # Ensure the returned object is a RosePicker with correct path - assert isinstance(rp, RosePicker) - assert Path(rp.exec_path) == rose_picker_path - - -def test_get_rose_picker_local_checkout_fails() -> None: - """ - This functions tests the behaviour if a local checkout fails, - i.e. rose_picker cannot be executed. This test patches the - ToolRepository (so that FCM is not actually called), and makes - sure RosePicker is always not available: - """ - - tag = "v2.0.0" - - # Make sure rose_picker will always return to be not available: - with patch("rose_picker_tool.ToolRepository.get_default") as mock_repo, \ - patch.object(RosePicker, "check_available", return_value=False), \ - pytest.raises(RuntimeError) as err: - get_rose_picker(tag) - - assert f"Cannot run rose_picker tag '{tag}'." == str(err.value) - # Also make sure that we indeed got FCM :) - mock_repo.assert_called_with(Category.FCM) +from rose_picker_tool import RosePicker def test_get_rose_picker_check_available() -> None: """ Test RosePicker's check_available. """ - rose_picker = RosePicker(Path("/usr/bin/rose_picker")) - with patch.object(RosePicker, "run", return_value=True) as mock_run: + rose_picker = RosePicker() + with patch("fab.tools.tool.Tool.run", return_value=True) as mock_run: assert rose_picker.check_available() - mock_run.assert_called_once_with(additional_parameters="-help") + mock_run.assert_called_once_with("-help") with patch.object(RosePicker, "run", side_effect=RuntimeError) as mock_run: assert not rose_picker.check_available() - mock_run.assert_called_once_with(additional_parameters="-help") + mock_run.assert_called_once_with("-help") def test_get_rose_picker_execute() -> None: """ Test RosePicker's check_available. """ - rose_picker = RosePicker(Path("/usr/bin/rose_picker")) - with patch.object(RosePicker, "run", return_value=0) as mock_run, \ + rose_picker = RosePicker() + rose_meta_conf = Path("rose_meta_conf") + directory = Path("/some/dir") + p1 = Path("/path1") + p2 = Path("/path12") + with patch("fab.tools.tool.Tool.run", return_value=0) as mock_run, \ patch.object(os, "environ", {}): - rose_picker.execute(["arg"]) + rose_picker.execute(rose_meta_conf, directory, include_paths=[p1, p2]) + # Rose picker prepends the existing python path, separated by ":". # Since python path is not set, there will be a leading ":"" - mock_run.assert_called_once_with(additional_parameters=["arg"], - env={'PYTHONPATH': ':/usr/lib/python'}) + mock_run.assert_called_once_with( + additional_parameters=[rose_meta_conf, "-directory", directory, + "-include_dirs", p1, "-include_dirs", p2, ]) From 6dc4bd89d42f36569ac28142f430f44dadd28738 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sun, 8 Feb 2026 11:25:48 +1100 Subject: [PATCH 14/95] Renamed test directory to tests (to avoid clash with .gitignore in root). --- infrastructure/build/fab/{test => tests}/configurator_test.py | 0 infrastructure/build/fab/{test => tests}/lfric_base_test.py | 0 infrastructure/build/fab/{test => tests}/rose_picker_tool_test.py | 0 infrastructure/build/fab/{test => tests}/templaterator_test.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename infrastructure/build/fab/{test => tests}/configurator_test.py (100%) rename infrastructure/build/fab/{test => tests}/lfric_base_test.py (100%) rename infrastructure/build/fab/{test => tests}/rose_picker_tool_test.py (100%) rename infrastructure/build/fab/{test => tests}/templaterator_test.py (100%) diff --git a/infrastructure/build/fab/test/configurator_test.py b/infrastructure/build/fab/tests/configurator_test.py similarity index 100% rename from infrastructure/build/fab/test/configurator_test.py rename to infrastructure/build/fab/tests/configurator_test.py diff --git a/infrastructure/build/fab/test/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py similarity index 100% rename from infrastructure/build/fab/test/lfric_base_test.py rename to infrastructure/build/fab/tests/lfric_base_test.py diff --git a/infrastructure/build/fab/test/rose_picker_tool_test.py b/infrastructure/build/fab/tests/rose_picker_tool_test.py similarity index 100% rename from infrastructure/build/fab/test/rose_picker_tool_test.py rename to infrastructure/build/fab/tests/rose_picker_tool_test.py diff --git a/infrastructure/build/fab/test/templaterator_test.py b/infrastructure/build/fab/tests/templaterator_test.py similarity index 100% rename from infrastructure/build/fab/test/templaterator_test.py rename to infrastructure/build/fab/tests/templaterator_test.py From 00b7ec96b8683d03bcbc7c5b2d1557360c7aa273 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 00:49:55 +1100 Subject: [PATCH 15/95] #240 Renamed rose_picker_tool.py to rose_picker.py etc. --- infrastructure/build/fab/configurator.py | 2 +- .../build/fab/{rose_picker_tool.py => rose_picker.py} | 0 infrastructure/build/fab/tests/configurator_test.py | 2 +- infrastructure/build/fab/tests/lfric_base_test.py | 9 +-------- .../{rose_picker_tool_test.py => rose_picker_test.py} | 2 +- 5 files changed, 4 insertions(+), 11 deletions(-) rename infrastructure/build/fab/{rose_picker_tool.py => rose_picker.py} (100%) rename infrastructure/build/fab/tests/{rose_picker_tool_test.py => rose_picker_test.py} (97%) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index fdea9a75a..21b2be0ce 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -17,7 +17,7 @@ from fab.api import BuildConfig, find_source_files, Category from fab.tools.shell import Shell -from rose_picker_tool import RosePicker +from rose_picker import RosePicker logger = logging.getLogger('fab') diff --git a/infrastructure/build/fab/rose_picker_tool.py b/infrastructure/build/fab/rose_picker.py similarity index 100% rename from infrastructure/build/fab/rose_picker_tool.py rename to infrastructure/build/fab/rose_picker.py diff --git a/infrastructure/build/fab/tests/configurator_test.py b/infrastructure/build/fab/tests/configurator_test.py index 12ca84698..1ab584cfb 100644 --- a/infrastructure/build/fab/tests/configurator_test.py +++ b/infrastructure/build/fab/tests/configurator_test.py @@ -61,7 +61,7 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): config_namelist.write_text("namelist1\nnamelist2\n", encoding="utf8") # Run configurator - with patch("rose_picker_tool.RosePicker.execute", + with patch("rose_picker.RosePicker.execute", return_value=0) as rose_picker, \ pytest.warns(match="_metric_send_conn not set, cannot " "send metrics"): diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py index 9ae1daef9..dfaed15d4 100644 --- a/infrastructure/build/fab/tests/lfric_base_test.py +++ b/infrastructure/build/fab/tests/lfric_base_test.py @@ -232,12 +232,10 @@ def test_command_line_options(monkeypatch) -> None: Tests LFRic specific command line options. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py", - "--rose_picker", "custom", "--precision-default", "32"]) lfric_base = LFRicBase(name="test") - assert lfric_base.args.rose_picker == "custom" assert lfric_base.args.precision_default == "32" @@ -437,12 +435,10 @@ def test_configurator_step(monkeypatch) -> None: # Create mock objects mock_config = mock.MagicMock() - mock_picker = mock.MagicMock(return_value="rose_picker_tool") mock_meta = mock.MagicMock(return_value="rose_meta.conf") # Set up mocks using monkeypatch monkeypatch.setattr('lfric_base.configurator', mock_config) - monkeypatch.setattr('lfric_base.get_rose_picker', mock_picker) lfric_base = LFRicBase(name="test") monkeypatch.setattr(lfric_base, 'get_rose_meta', mock_meta) @@ -455,7 +451,6 @@ def test_configurator_step(monkeypatch) -> None: lfric_core_source=lfric_base.lfric_core_root, rose_meta_conf="rose_meta.conf", include_paths=[], - rose_picker="rose_picker_tool" ) @@ -665,13 +660,11 @@ def test_psyclone_step(monkeypatch) -> None: lfric_base.psyclone_step(additional_parameters=["-additional"]) # Verify psyclone called with correct arguments - print(mock_psy.mock_calls) - print("UUU", mock_config_opts, mock_additional_opts) mock_psy.assert_called_once_with( lfric_base.config, kernel_roots=[(lfric_base.config.build_output / "kernel")], transformation_script=lfric_base.get_transformation_script, - api="dynamo0.3", + api="lfric", cli_args=mock_config_opts + mock_additional_opts + ["-additional"], ignore_dependencies=None ) diff --git a/infrastructure/build/fab/tests/rose_picker_tool_test.py b/infrastructure/build/fab/tests/rose_picker_test.py similarity index 97% rename from infrastructure/build/fab/tests/rose_picker_tool_test.py rename to infrastructure/build/fab/tests/rose_picker_test.py index 722ac9106..e1bc1252c 100644 --- a/infrastructure/build/fab/tests/rose_picker_tool_test.py +++ b/infrastructure/build/fab/tests/rose_picker_test.py @@ -13,7 +13,7 @@ from pathlib import Path from unittest.mock import patch -from rose_picker_tool import RosePicker +from rose_picker import RosePicker def test_get_rose_picker_check_available() -> None: From 6a2519509caaf773ef60a65be84da3633a79a4b2 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 00:52:44 +1100 Subject: [PATCH 16/95] #240 Addressed issues raised in review. --- infrastructure/build/fab/lfric_base.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 0b83cb097..9dbc0dbdf 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -171,13 +171,11 @@ def define_preprocessor_flags_step(self) -> None: continue # No command line option for the current precision name. - # Check if a default was set (--precision-default) - if generic_default: - preprocessor_flags.append(f"-D{prec_name}=" - f"{generic_default}") - else: - # Otherwise, use the default for this precision - preprocessor_flags.append(f"-D{prec_name}={prec_default}") + # Check if a default was set (--precision-default), otherwise + # use the default for this precision + preprocessor_flags.append( + f"-D{prec_name}=" + f"{generic_default if generic_default else prec_default}") # core/components/lfric-xios/build/import.mk if not self.args.no_xios: @@ -235,7 +233,6 @@ def find_source_files_step( :param path_filters: optional list of path filters to be passed to Fab find_source_files, default is None. - :type path_filters: Optional[Iterable[Exclude, Include]] ''' self.configurator_step() @@ -371,7 +368,7 @@ def psyclone_step( psyclone(self.config, kernel_roots=[(self.config.build_output / "kernel")], transformation_script=self.get_transformation_script, - api="dynamo0.3", + api="lfric", cli_args=psyclone_cli_args, ignore_dependencies=ignore_dependencies) sys.path = old_sys_path From 4872ec9427ae8ba92add42d55b33c09dd76ef795 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 11:24:09 +1100 Subject: [PATCH 17/95] #240 Removed get_additional_psyclone_options. --- infrastructure/build/fab/lfric_base.py | 33 +++++++------------ .../build/fab/tests/lfric_base_test.py | 17 ++-------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 9dbc0dbdf..0e2b4c100 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -61,8 +61,6 @@ def __init__(self, name: str, if root_symbol: self.set_root_symbol(root_symbol) - self._psyclone_config = (self.config.source_root / 'psyclone_config' / - 'psyclone.cfg') # Many PSyclone scripts use module(s) from this directory. Additional # paths might need to be added later. self._add_python_paths = [str(self.lfric_core_root / "infrastructure" / @@ -343,22 +341,19 @@ def psyclone_step( additional_parameters: Optional[list[str]] = None ) -> None: ''' - This method runs Fab's psyclone. It first sets the additional psyclone + This method runs Fab's psyclone. It first sets the psyclone command line arguments by calling get_psyclone_config to get the - PSyclone configuration file and by calling - `get_additional_psyclone_options` to get additional psyclone command - line set by the user, e.g. for profiling, if any. Finally, Fab's - psyclone is called with the Fab build configuration, the kernel root - directory, the transformation script got through calling - `get_transformation_script`, the api, and the additional psyclone - command line arguments. + PSyclone configuration file. Additional flags can be set in the + PSyclone tool. Finally, Fab's psyclone is called with the Fab build + configuration, the kernel root directory, the transformation script + got through calling `get_transformation_script`, the api, and the + additional psyclone command line arguments. :param ignore_dependencies: :param additional_parameters: optional additional parameter for the PSyclone. ''' - psyclone_cli_args = self.get_psyclone_config() - psyclone_cli_args.extend(self.get_additional_psyclone_options()) + psyclone_cli_args = ["--config", self.get_psyclone_config()] if additional_parameters: psyclone_cli_args.extend(additional_parameters) @@ -373,18 +368,12 @@ def psyclone_step( ignore_dependencies=ignore_dependencies) sys.path = old_sys_path - def get_psyclone_config(self) -> List[str]: + def get_psyclone_config(self) -> str: ''' - :returns: the command line options to pick the right - PSyclone config file. + :returns: the PSyclone config file as string. ''' - return ["--config", str(self._psyclone_config)] - - def get_additional_psyclone_options(self) -> List[str]: - ''' - A placeholder for additional PSyclone comand line options. - ''' - return [] + return str(self.config.source_root / 'psyclone_config' / + 'psyclone.cfg') def get_transformation_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py index dfaed15d4..fd755ae58 100644 --- a/infrastructure/build/fab/tests/lfric_base_test.py +++ b/infrastructure/build/fab/tests/lfric_base_test.py @@ -653,8 +653,6 @@ def test_psyclone_step(monkeypatch) -> None: # PSyclone modified these lists in the lambdas when it modifies the list monkeypatch.setattr(lfric_base, 'get_psyclone_config', lambda: mock_config_opts[:]) - monkeypatch.setattr(lfric_base, 'get_additional_psyclone_options', - lambda: mock_additional_opts[:]) # Call method under test lfric_base.psyclone_step(additional_parameters=["-additional"]) @@ -679,19 +677,8 @@ def test_get_psyclone_config(monkeypatch) -> None: lfric_base = LFRicBase(name="test") config_args = lfric_base.get_psyclone_config() - assert config_args == ["--config", - str(lfric_base.config.source_root / - 'psyclone_config/psyclone.cfg')] - - -def test_get_additional_psyclone_options(monkeypatch) -> None: - ''' - Tests getting additional PSyclone options (for profiling). - ''' - monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - - lfric_base = LFRicBase(name="test") - assert not lfric_base.get_additional_psyclone_options() + assert config_args == str(lfric_base.config.source_root / + 'psyclone_config/psyclone.cfg') def test_get_transformation_script(monkeypatch, tmp_path) -> None: From 495d651e7008b99b27fffd6164d729a4cda9c9aa Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 11:32:34 +1100 Subject: [PATCH 18/95] #240 Fixed configurator to use the new names of the tools. --- infrastructure/build/fab/configurator.py | 4 ++-- infrastructure/build/fab/tests/configurator_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index 21b2be0ce..143a7a96a 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -65,7 +65,7 @@ def configurator(config: BuildConfig, # -------------------- # builds a bunch of f90s from the json logger.info('GenerateNamelist') - shell.exec(f"{tools / 'GenerateNamelist'} -verbose {rose_meta} " + shell.exec(f"{tools / 'GenerateNamelistLoader'} -verbose {rose_meta} " f"-directory {config_dir}") # create configuration_mod.f90 in source root @@ -75,7 +75,7 @@ def configurator(config: BuildConfig, names = [name.strip() for name in f_in.readlines()] configuration_mod_fpath = config_dir / 'configuration_mod.f90' - shell.exec(f"{tools / 'GenerateLoader'} {configuration_mod_fpath} " + shell.exec(f"{tools / 'GenerateConfigLoader'} {configuration_mod_fpath} " f"{' '.join(names)}") # create feign_config_mod.f90 in source root diff --git a/infrastructure/build/fab/tests/configurator_test.py b/infrastructure/build/fab/tests/configurator_test.py index 1ab584cfb..ade206a61 100644 --- a/infrastructure/build/fab/tests/configurator_test.py +++ b/infrastructure/build/fab/tests/configurator_test.py @@ -84,10 +84,10 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): # Check shell.exec was called with expected commands expected_calls = [ - ((f"{tools_dir / 'GenerateNamelist'} " + ((f"{tools_dir / 'GenerateNamelistLoader'} " f"-verbose {config_dir / 'rose-meta.json'} " f"-directory {config_dir}"),), - ((f"{tools_dir / 'GenerateLoader'} " + ((f"{tools_dir / 'GenerateConfigLoader'} " f"{config_dir / 'configuration_mod.f90'} " f"namelist1 namelist2"),), ((f"{tools_dir / 'GenerateFeigns'} {config_dir / 'rose-meta.json'} " From fa846628429ee785980dd5e070ae190f672b94a5 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 17:15:08 +1100 Subject: [PATCH 19/95] #240 Fixed building current LFRic skeleton due to changes in build system and scripts. --- infrastructure/build/fab/configurator.py | 23 +++++++++++++++++++---- infrastructure/build/fab/lfric_base.py | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index 143a7a96a..6feb3ea2e 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -70,13 +70,28 @@ def configurator(config: BuildConfig, # create configuration_mod.f90 in source root # ------------------------------------------- - logger.info('GenerateLoader') + logger.info('GenerateConfigLoader') with open(config_dir / 'config_namelists.txt', encoding="utf8") as f_in: names = [name.strip() for name in f_in.readlines()] - configuration_mod_fpath = config_dir / 'configuration_mod.f90' - shell.exec(f"{tools / 'GenerateConfigLoader'} {configuration_mod_fpath} " - f"{' '.join(names)}") + shell.exec(f"{tools / 'GenerateConfigLoader'} " + f"{' '.join(names)} " + f"-o {config_dir}") + + logger.info('GenerateExtendedNamelistType') + shell.exec(f"{tools / 'GenerateExtendedNamelistType'} {rose_meta} " + f"-directory {config_dir}") + + duplicates: list[str] = [] + with open(config_dir / 'duplicate_namelists.txt', encoding="utf8") as f_in: + for name in f_in.readlines(): + duplicates.extend(["-duplicate", name.strip()]) + + logger.info('GenerateConfigType') + shell.exec(f"{tools / 'GenerateConfigType'} " + f"{' '.join(names)} " + f"{' '.join(duplicates)} " + f"-o {config_dir}") # create feign_config_mod.f90 in source root # ------------------------------------------ diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 0e2b4c100..02eed5131 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -385,6 +385,7 @@ def get_transformation_script(self, fpath: Path, :param fpath: the path to the file being processed. :param config: the FAB BuildConfig instance. + :returns: the transformation script to be used by PSyclone. ''' # Newer LFRic versions have a psykal directory From bf87398f4fe0bd1bcff5d4bb90c465f0f2375bd1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 10 Feb 2026 17:29:30 +1100 Subject: [PATCH 20/95] #240 Updated tests to work with changes to LFRic build system. --- infrastructure/build/fab/configurator.py | 2 +- infrastructure/build/fab/tests/configurator_test.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index 6feb3ea2e..dbd14d74e 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -64,7 +64,7 @@ def configurator(config: BuildConfig, # build_config_loaders # -------------------- # builds a bunch of f90s from the json - logger.info('GenerateNamelist') + logger.info('GenerateNamelistLoader') shell.exec(f"{tools / 'GenerateNamelistLoader'} -verbose {rose_meta} " f"-directory {config_dir}") diff --git a/infrastructure/build/fab/tests/configurator_test.py b/infrastructure/build/fab/tests/configurator_test.py index ade206a61..5eb134934 100644 --- a/infrastructure/build/fab/tests/configurator_test.py +++ b/infrastructure/build/fab/tests/configurator_test.py @@ -60,6 +60,10 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): config_namelist = config_dir / "config_namelists.txt" config_namelist.write_text("namelist1\nnamelist2\n", encoding="utf8") + # Simulate duplicate_namelist.txt: + duplicate_namelist = config_dir / "duplicate_namelists.txt" + duplicate_namelist.write_text("duplicate1\nduplicate2\n", encoding="utf8") + # Run configurator with patch("rose_picker.RosePicker.execute", return_value=0) as rose_picker, \ @@ -88,8 +92,13 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): f"-verbose {config_dir / 'rose-meta.json'} " f"-directory {config_dir}"),), ((f"{tools_dir / 'GenerateConfigLoader'} " - f"{config_dir / 'configuration_mod.f90'} " - f"namelist1 namelist2"),), + f"namelist1 namelist2 -o {config_dir}"),), + ((f"{tools_dir / 'GenerateExtendedNamelistType'} " + f"{config_dir / 'rose-meta.json'} " + f"-directory {config_dir}"),), + ((f"{tools_dir / 'GenerateConfigType'} " + f"namelist1 namelist2 -duplicate duplicate1 -duplicate duplicate2 " + f"-o {config_dir}"),), ((f"{tools_dir / 'GenerateFeigns'} {config_dir / 'rose-meta.json'} " f"-output {config_dir / 'feign_config_mod.f90'}"),) ] From 463faaeda9a3de90eefe98c5952c8be01ea6b2d7 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 11 Feb 2026 10:57:32 +1100 Subject: [PATCH 21/95] #240 Removed unnecessary path for rose picker. --- infrastructure/build/fab/configurator.py | 4 ++-- infrastructure/build/fab/tests/configurator_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/build/fab/configurator.py b/infrastructure/build/fab/configurator.py index dbd14d74e..412c95d82 100755 --- a/infrastructure/build/fab/configurator.py +++ b/infrastructure/build/fab/configurator.py @@ -48,10 +48,10 @@ def configurator(config: BuildConfig, # gungho/build logger.info('rose_picker') - include_dirs = [lfric_core_source, lfric_core_source / 'rose-meta'] + include_dirs = [lfric_core_source / 'rose-meta'] if include_paths: for path in include_paths: - include_dirs.extend([path, path / 'rose-meta']) + include_dirs.append(path / 'rose-meta') rose_picker = RosePicker() rose_picker.execute(rose_meta_conf, config_dir, diff --git a/infrastructure/build/fab/tests/configurator_test.py b/infrastructure/build/fab/tests/configurator_test.py index 5eb134934..49b8d5426 100644 --- a/infrastructure/build/fab/tests/configurator_test.py +++ b/infrastructure/build/fab/tests/configurator_test.py @@ -83,8 +83,8 @@ def test_configurator_runs_expected_sequence(mock_shell, tmp_path): rose_picker.assert_called_once() rose_picker.assert_called_with( rose_meta_conf, config_dir, - include_paths=[lfric_core, lfric_core / "rose-meta", - lfric_apps, lfric_apps / "rose-meta"]) + include_paths=[lfric_core / "rose-meta", + lfric_apps / "rose-meta"]) # Check shell.exec was called with expected commands expected_calls = [ From 0c9a5997a88c5acd852ebdef46a96138ce76301f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 11 Feb 2026 10:58:54 +1100 Subject: [PATCH 22/95] #240 Set a default as project name. --- applications/skeleton/fab_skeleton.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index 11df842c1..dd31ea1ed 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -35,7 +35,7 @@ class FabSkeleton(LFRicBase): :param name: The name of the application. """ - def __init__(self, name: str) -> None: + def __init__(self, name: str = "skeleton") -> None: super().__init__(name=name) # Store the root of this apps for later this_file = Path(__file__).resolve() @@ -70,5 +70,5 @@ def get_rose_meta(self) -> Path: logger = logging.getLogger('fab') logger.setLevel(logging.DEBUG) - fab_skeleton = FabSkeleton(name="skeleton") + fab_skeleton = FabSkeleton() fab_skeleton.build() From ac0999cc928ca8455108ba01589367844e82722e Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 11 Feb 2026 17:25:42 +1100 Subject: [PATCH 23/95] #240 Fixed incorrect names in templaterator. --- infrastructure/build/fab/lfric_base.py | 14 +++++---- .../build/fab/tests/lfric_base_test.py | 30 +++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 02eed5131..3f2e4f017 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -278,18 +278,22 @@ def templaterator_step(self, config: BuildConfig) -> None: t90_filter = SuffixFilter(ArtefactSet.INITIAL_SOURCE_FILES, [".t90", ".T90"]) template_files = t90_filter(config.artefact_store) - # Don't bother with parallelising this, atm there is only one file: + templ_r32 = {"kind": "real32", "type": "real"} + templ_r64 = {"kind": "real64", "type": "real"} + templ_i32 = {"kind": "int32", "type": "integer"} + # Don't bother with parallelising this, it's fast for template_file in template_files: out_dir = input_to_output_fpath(config=config, input_path=template_file).parent out_dir.mkdir(parents=True, exist_ok=True) - templ_r32 = {"kind": "real32", "type": "real"} - templ_r64 = {"kind": "real64", "type": "real"} - templ_i32 = {"kind": "int32", "type": "integer"} + template_stem = template_file.stem.removesuffix("_mod") for key_values in [templ_r32, templ_r64, templ_i32]: - out_file = out_dir / f"field_{key_values['kind']}_mod.f90" + out_file = (out_dir / + f"{template_stem}_{key_values['kind']}_mod.f90") templaterator.process(template_file, out_file, key_values=key_values) + # Add the newly created file to the set of + # Fortran files to compile config.artefact_store.add(ArtefactSet.FORTRAN_COMPILER_FILES, out_file) diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py index fd755ae58..298de0f79 100644 --- a/infrastructure/build/fab/tests/lfric_base_test.py +++ b/infrastructure/build/fab/tests/lfric_base_test.py @@ -461,7 +461,9 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) # Create mock template file - template_file = tmp_path / "field.t90" + source_path = tmp_path / "source" + source_path.mkdir(parents=True) + template_file = source_path / "field.t90" template_file.write_text("template content", encoding='utf-8') # Create mock templaterator @@ -471,16 +473,8 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: monkeypatch.setattr('lfric_base.Templaterator', mock_templaterator) # Mock input_to_output_fpath - mock_output_path = tmp_path / "build" / "output" + mock_output_path = tmp_path / "build_output" mock_output_path.mkdir(parents=True) - monkeypatch.setattr('lfric_base.input_to_output_fpath', - lambda config, input_path: (mock_output_path / - input_path.name)) - - # Mock SuffixFilter to return our template file - mock_filter = mock.MagicMock() - mock_filter.return_value = {template_file} - monkeypatch.setattr('lfric_base.SuffixFilter', lambda *args: mock_filter) # Create mock config with proper artefact store mock_artefact_store = mock.MagicMock() @@ -488,7 +482,13 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: config = mock.MagicMock() config.artefact_store = mock_artefact_store - config.build_output = tmp_path + config.build_output = mock_output_path + config.source_root = source_path + + # Mock SuffixFilter to return our template file + mock_filter = mock.MagicMock() + mock_filter.return_value = {template_file} + monkeypatch.setattr('lfric_base.SuffixFilter', lambda *args: mock_filter) # Create LFRicBase instance lfric_base = LFRicBase(name="test") @@ -511,6 +511,7 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: ] for template in templates: + out_file = mock_output_path / f"field_{template['kind']}_mod.f90" out_file = mock_output_path / f"field_{template['kind']}_mod.f90" expected_calls.append( mock.call(template_file, out_file, key_values=template) @@ -641,8 +642,7 @@ def test_psyclone_step(monkeypatch) -> None: # Create mock objects mock_psy = mock.MagicMock() - mock_config_opts = ["--config", "/mock/psyclone.cfg"] - mock_additional_opts: List[str] = [] + mock_psyclone_config = "/mock/psyclone.cfg" # Set up monkeypatch for module level import monkeypatch.setattr('lfric_base.psyclone', mock_psy) @@ -652,7 +652,7 @@ def test_psyclone_step(monkeypatch) -> None: # Patch instance methods. Return a copy to avoid that # PSyclone modified these lists in the lambdas when it modifies the list monkeypatch.setattr(lfric_base, 'get_psyclone_config', - lambda: mock_config_opts[:]) + lambda: mock_psyclone_config) # Call method under test lfric_base.psyclone_step(additional_parameters=["-additional"]) @@ -663,7 +663,7 @@ def test_psyclone_step(monkeypatch) -> None: kernel_roots=[(lfric_base.config.build_output / "kernel")], transformation_script=lfric_base.get_transformation_script, api="lfric", - cli_args=mock_config_opts + mock_additional_opts + ["-additional"], + cli_args=(["--config", mock_psyclone_config, "-additional"]), ignore_dependencies=None ) From c0fc24f17b823de9ad699a6b4d580952f308e7aa Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 19 Feb 2026 15:42:05 +1100 Subject: [PATCH 24/95] #240 Removed support for old-style environment variables for precision. --- infrastructure/build/fab/lfric_base.py | 6 ------ infrastructure/build/fab/tests/lfric_base_test.py | 5 +++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 3f2e4f017..7a6de2462 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -13,7 +13,6 @@ """ import argparse -import os from pathlib import Path import sys from typing import List, Optional, Iterable, Union @@ -162,11 +161,6 @@ def define_preprocessor_flags_step(self) -> None: value = getattr(self.args, prec_name.lower()) preprocessor_flags.append(f"-D{prec_name}={value}") continue - # Check for environment variable which can overwrite the default: - env_precision = os.environ.get(prec_name) - if env_precision: - preprocessor_flags.append(f"-D{prec_name}={env_precision}") - continue # No command line option for the current precision name. # Check if a default was set (--precision-default), otherwise diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py index 298de0f79..f421bbeea 100644 --- a/infrastructure/build/fab/tests/lfric_base_test.py +++ b/infrastructure/build/fab/tests/lfric_base_test.py @@ -286,8 +286,9 @@ def test_precision_definition_with_default(monkeypatch) -> None: assert '-DR_SOLVER_PRECISION=32' in flags # Specified default of any precision assert '-DR_TRAN_PRECISION=32' in flags - # From environment variable - assert '-DR_BL_PRECISION=64' in flags + # Old style environment variables must be ignored, so R_BL_PRECISION + # must still be 32! + assert '-DR_BL_PRECISION=32' in flags @pytest.mark.parametrize('no_xios', [True, False]) From 434c394bd3ca2c41a9a603f33af12686c1990bad Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 19 Feb 2026 15:42:17 +1100 Subject: [PATCH 25/95] #240 Removed unnecessary import. --- infrastructure/build/fab/lfric_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 7a6de2462..cd869e30b 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -133,7 +133,6 @@ def setup_site_specific_location(self): baf base would set up). ''' this_dir = Path(__file__).parent - sys.path.insert(0, str(this_dir)) # We need to add the 'site_specific' directory to the path, so # each config can import from 'default' (instead of having to # use 'site_specific.default', which would hard-code the name From 890a72527b5a0be9f0d2b8bbb77929656c866e02 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 19 Feb 2026 19:26:37 +1100 Subject: [PATCH 26/95] #240 Marked the new steps as steps so they get measured. --- infrastructure/build/fab/lfric_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index cd869e30b..5e379b5a6 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -18,7 +18,7 @@ from typing import List, Optional, Iterable, Union from fab.api import (ArtefactSet, BuildConfig, Exclude, grab_folder, Include, - input_to_output_fpath, preprocess_x90, psyclone, + input_to_output_fpath, preprocess_x90, psyclone, step, SuffixFilter) from fab.fab_base.fab_base import FabBase @@ -233,6 +233,7 @@ def find_source_files_step( self.templaterator_step(self.config) + @step def configurator_step( self, include_paths: Optional[list[Path]] = None) -> None: @@ -257,6 +258,7 @@ def configurator_step( rose_meta_conf=rose_meta, include_paths=include_paths) + @step def templaterator_step(self, config: BuildConfig) -> None: ''' This method runs the LFRic templaterator Fab tool. From d98699b85e29fe49b04be5dd3864df486a55223f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 19 Feb 2026 19:26:56 +1100 Subject: [PATCH 27/95] #240 Clarified docstring. --- infrastructure/build/fab/lfric_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index 5e379b5a6..f187991b2 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -369,6 +369,10 @@ def psyclone_step( def get_psyclone_config(self) -> str: ''' + This method can be overwritten if an application needs to provide + a modified psyclone config file (e.g. to enable additional + debug options). + :returns: the PSyclone config file as string. ''' return str(self.config.source_root / 'psyclone_config' / From 5f2c8927aa0864e0fd9f66b0bc421b22bd955c2a Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 20 Feb 2026 16:54:34 +1100 Subject: [PATCH 28/95] Updated comments. --- infrastructure/build/fab/lfric_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/infrastructure/build/fab/lfric_base.py b/infrastructure/build/fab/lfric_base.py index f187991b2..e120fdccc 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/infrastructure/build/fab/lfric_base.py @@ -395,17 +395,26 @@ def get_transformation_script(self, fpath: Path, optimisation_path = (config.source_root / "optimisation" / f"{self.site}-{self.platform}" / "psykal") relative_path = None + # The soure file might be either in build_output (e.g. a preprocessed + # .X90 file), or still in source (.x90 file). Check if the file + # is in one of the two sub-trees, and use the relative path to + # check if there is a file-specific optimisation script for base_path in [config.source_root, config.build_output]: try: relative_path = fpath.relative_to(base_path) except ValueError: + # The file is not under the `base_path` - keep on checking pass + if relative_path: + # The file was under either source or build. Check if there + # is a file-specific optimisation script: local_transformation_script = (optimisation_path / (relative_path.with_suffix('.py'))) if local_transformation_script.exists(): return local_transformation_script + # No file-specific optimisation script found. Check for global.py: global_transformation_script = optimisation_path / 'global.py' if global_transformation_script.exists(): return global_transformation_script From 3c3ba13c90a5255a5d4a485c05339584dff49e58 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 20 Feb 2026 17:30:21 +1100 Subject: [PATCH 29/95] #240 Handle warning in new steps. --- infrastructure/build/fab/tests/lfric_base_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/infrastructure/build/fab/tests/lfric_base_test.py index f421bbeea..aa839a2d5 100644 --- a/infrastructure/build/fab/tests/lfric_base_test.py +++ b/infrastructure/build/fab/tests/lfric_base_test.py @@ -496,7 +496,8 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: monkeypatch.setattr(lfric_base, '_lfric_core_root', tmp_path) # Run templaterator step - lfric_base.templaterator_step(config) + with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): + lfric_base.templaterator_step(config) # Verify templaterator initialization mock_templaterator.assert_called_once_with(tmp_path / "infrastructure" / @@ -510,7 +511,6 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: {"kind": "real64", "type": "real"}, {"kind": "int32", "type": "integer"} ] - for template in templates: out_file = mock_output_path / f"field_{template['kind']}_mod.f90" out_file = mock_output_path / f"field_{template['kind']}_mod.f90" @@ -534,7 +534,9 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: # Test empty template files case mock_filter.return_value = set() - lfric_base.templaterator_step(config) + + with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): + lfric_base.templaterator_step(config) # Call count should remain the same since no new files processed assert mock_templaterator_instance.process.call_count == 3 From d16a46be6685ecbc9f4d649ed53072f2063c0840 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 20 Feb 2026 18:00:44 +1100 Subject: [PATCH 30/95] #240 Moved build files into lfric_build directory. --- applications/skeleton/fab_skeleton.py | 3 +-- {infrastructure/build/fab => lfric_build}/README.md | 0 {infrastructure/build/fab => lfric_build}/configurator.py | 0 {infrastructure/build/fab => lfric_build}/lfric_base.py | 2 +- {infrastructure/build/fab => lfric_build}/rose_picker.py | 0 .../fab => lfric_build}/site_specific/default/__init__.py | 0 .../build/fab => lfric_build}/site_specific/default/config.py | 0 .../fab => lfric_build}/site_specific/default/setup_cray.py | 0 .../fab => lfric_build}/site_specific/default/setup_gnu.py | 0 .../site_specific/default/setup_intel_classic.py | 0 .../site_specific/default/setup_intel_llvm.py | 0 .../fab => lfric_build}/site_specific/default/setup_nvidia.py | 0 .../fab => lfric_build}/site_specific/meto_ex1a/config.py | 0 .../build/fab => lfric_build}/site_specific/ncas_ex/config.py | 0 .../fab => lfric_build}/site_specific/nci_gadi/__init__.py | 0 .../build/fab => lfric_build}/site_specific/nci_gadi/config.py | 0 .../fab => lfric_build}/site_specific/niwa_xc50/config.py | 0 {infrastructure/build/fab => lfric_build}/templaterator.py | 0 .../build/fab => lfric_build}/tests/configurator_test.py | 0 .../build/fab => lfric_build}/tests/lfric_base_test.py | 0 .../build/fab => lfric_build}/tests/rose_picker_test.py | 0 .../build/fab => lfric_build}/tests/templaterator_test.py | 0 22 files changed, 2 insertions(+), 3 deletions(-) rename {infrastructure/build/fab => lfric_build}/README.md (100%) rename {infrastructure/build/fab => lfric_build}/configurator.py (100%) rename {infrastructure/build/fab => lfric_build}/lfric_base.py (99%) rename {infrastructure/build/fab => lfric_build}/rose_picker.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/__init__.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/config.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/setup_cray.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/setup_gnu.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/setup_intel_classic.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/setup_intel_llvm.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/default/setup_nvidia.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/meto_ex1a/config.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/ncas_ex/config.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/nci_gadi/__init__.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/nci_gadi/config.py (100%) rename {infrastructure/build/fab => lfric_build}/site_specific/niwa_xc50/config.py (100%) rename {infrastructure/build/fab => lfric_build}/templaterator.py (100%) rename {infrastructure/build/fab => lfric_build}/tests/configurator_test.py (100%) rename {infrastructure/build/fab => lfric_build}/tests/lfric_base_test.py (100%) rename {infrastructure/build/fab => lfric_build}/tests/rose_picker_test.py (100%) rename {infrastructure/build/fab => lfric_build}/tests/templaterator_test.py (100%) diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index dd31ea1ed..63596dab2 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -20,8 +20,7 @@ from fab.steps.grab.folder import grab_folder # We need to import the base class: -sys.path.insert(0, str(Path(__file__).parents[2] / "infrastructure" / - "build" / "fab")) +sys.path.insert(0, str(Path(__file__).parents[2] / "lfric_build")) from lfric_base import LFRicBase # noqa: E402 diff --git a/infrastructure/build/fab/README.md b/lfric_build/README.md similarity index 100% rename from infrastructure/build/fab/README.md rename to lfric_build/README.md diff --git a/infrastructure/build/fab/configurator.py b/lfric_build/configurator.py similarity index 100% rename from infrastructure/build/fab/configurator.py rename to lfric_build/configurator.py diff --git a/infrastructure/build/fab/lfric_base.py b/lfric_build/lfric_base.py similarity index 99% rename from infrastructure/build/fab/lfric_base.py rename to lfric_build/lfric_base.py index e120fdccc..3b6e99bbc 100755 --- a/infrastructure/build/fab/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -53,7 +53,7 @@ def __init__(self, name: str, this_file = Path(__file__) # The root directory of the LFRic Core - self._lfric_core_root = this_file.parents[3] + self._lfric_core_root = this_file.parents[1] # If the user wants to overwrite the default root symbol (which # is `name`): diff --git a/infrastructure/build/fab/rose_picker.py b/lfric_build/rose_picker.py similarity index 100% rename from infrastructure/build/fab/rose_picker.py rename to lfric_build/rose_picker.py diff --git a/infrastructure/build/fab/site_specific/default/__init__.py b/lfric_build/site_specific/default/__init__.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/__init__.py rename to lfric_build/site_specific/default/__init__.py diff --git a/infrastructure/build/fab/site_specific/default/config.py b/lfric_build/site_specific/default/config.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/config.py rename to lfric_build/site_specific/default/config.py diff --git a/infrastructure/build/fab/site_specific/default/setup_cray.py b/lfric_build/site_specific/default/setup_cray.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/setup_cray.py rename to lfric_build/site_specific/default/setup_cray.py diff --git a/infrastructure/build/fab/site_specific/default/setup_gnu.py b/lfric_build/site_specific/default/setup_gnu.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/setup_gnu.py rename to lfric_build/site_specific/default/setup_gnu.py diff --git a/infrastructure/build/fab/site_specific/default/setup_intel_classic.py b/lfric_build/site_specific/default/setup_intel_classic.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/setup_intel_classic.py rename to lfric_build/site_specific/default/setup_intel_classic.py diff --git a/infrastructure/build/fab/site_specific/default/setup_intel_llvm.py b/lfric_build/site_specific/default/setup_intel_llvm.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/setup_intel_llvm.py rename to lfric_build/site_specific/default/setup_intel_llvm.py diff --git a/infrastructure/build/fab/site_specific/default/setup_nvidia.py b/lfric_build/site_specific/default/setup_nvidia.py similarity index 100% rename from infrastructure/build/fab/site_specific/default/setup_nvidia.py rename to lfric_build/site_specific/default/setup_nvidia.py diff --git a/infrastructure/build/fab/site_specific/meto_ex1a/config.py b/lfric_build/site_specific/meto_ex1a/config.py similarity index 100% rename from infrastructure/build/fab/site_specific/meto_ex1a/config.py rename to lfric_build/site_specific/meto_ex1a/config.py diff --git a/infrastructure/build/fab/site_specific/ncas_ex/config.py b/lfric_build/site_specific/ncas_ex/config.py similarity index 100% rename from infrastructure/build/fab/site_specific/ncas_ex/config.py rename to lfric_build/site_specific/ncas_ex/config.py diff --git a/infrastructure/build/fab/site_specific/nci_gadi/__init__.py b/lfric_build/site_specific/nci_gadi/__init__.py similarity index 100% rename from infrastructure/build/fab/site_specific/nci_gadi/__init__.py rename to lfric_build/site_specific/nci_gadi/__init__.py diff --git a/infrastructure/build/fab/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py similarity index 100% rename from infrastructure/build/fab/site_specific/nci_gadi/config.py rename to lfric_build/site_specific/nci_gadi/config.py diff --git a/infrastructure/build/fab/site_specific/niwa_xc50/config.py b/lfric_build/site_specific/niwa_xc50/config.py similarity index 100% rename from infrastructure/build/fab/site_specific/niwa_xc50/config.py rename to lfric_build/site_specific/niwa_xc50/config.py diff --git a/infrastructure/build/fab/templaterator.py b/lfric_build/templaterator.py similarity index 100% rename from infrastructure/build/fab/templaterator.py rename to lfric_build/templaterator.py diff --git a/infrastructure/build/fab/tests/configurator_test.py b/lfric_build/tests/configurator_test.py similarity index 100% rename from infrastructure/build/fab/tests/configurator_test.py rename to lfric_build/tests/configurator_test.py diff --git a/infrastructure/build/fab/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py similarity index 100% rename from infrastructure/build/fab/tests/lfric_base_test.py rename to lfric_build/tests/lfric_base_test.py diff --git a/infrastructure/build/fab/tests/rose_picker_test.py b/lfric_build/tests/rose_picker_test.py similarity index 100% rename from infrastructure/build/fab/tests/rose_picker_test.py rename to lfric_build/tests/rose_picker_test.py diff --git a/infrastructure/build/fab/tests/templaterator_test.py b/lfric_build/tests/templaterator_test.py similarity index 100% rename from infrastructure/build/fab/tests/templaterator_test.py rename to lfric_build/tests/templaterator_test.py From d3d4673b679aaed8a2f9557bf4647a44d7007729 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 23 Feb 2026 21:22:54 +1100 Subject: [PATCH 31/95] #240 Renamed default compiler setup scripts to 'setup_script_XXX.py'. --- lfric_build/site_specific/default/config.py | 20 +++++++++---------- .../{setup_cray.py => setup_script_cray.py} | 3 ++- .../{setup_gnu.py => setup_script_gnu.py} | 3 ++- ...assic.py => setup_script_intel_classic.py} | 4 ++-- ...tel_llvm.py => setup_script_intel_llvm.py} | 4 ++-- ...setup_nvidia.py => setup_script_nvidia.py} | 3 ++- 6 files changed, 20 insertions(+), 17 deletions(-) rename lfric_build/site_specific/default/{setup_cray.py => setup_script_cray.py} (97%) rename lfric_build/site_specific/default/{setup_gnu.py => setup_script_gnu.py} (97%) rename lfric_build/site_specific/default/{setup_intel_classic.py => setup_script_intel_classic.py} (97%) rename lfric_build/site_specific/default/{setup_intel_llvm.py => setup_script_intel_llvm.py} (96%) rename lfric_build/site_specific/default/{setup_nvidia.py => setup_script_nvidia.py} (96%) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 74f1291bd..a2338bf51 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -10,11 +10,11 @@ from fab.api import AddFlags, BuildConfig, Category, ToolRepository -from default.setup_cray import setup_cray -from default.setup_gnu import setup_gnu -from default.setup_intel_classic import setup_intel_classic -from default.setup_intel_llvm import setup_intel_llvm -from default.setup_nvidia import setup_nvidia +from default.setup_script_cray import setup_script_cray +from default.setup_script_gnu import setup_script_gnu +from default.setup_script_intel_classic import setup_script_intel_classic +from default.setup_script_intel_llvm import setup_script_intel_llvm +from default.setup_script_nvidia import setup_script_nvidia class Config: @@ -104,7 +104,7 @@ def setup_cray(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - setup_cray(build_config, self.args) + setup_script_cray(build_config, self.args) def setup_gnu(self, build_config: BuildConfig) -> None: ''' @@ -116,7 +116,7 @@ def setup_gnu(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - setup_gnu(build_config, self.args) + setup_script_gnu(build_config, self.args) def setup_intel_classic(self, build_config: BuildConfig) -> None: ''' @@ -128,7 +128,7 @@ def setup_intel_classic(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - setup_intel_classic(build_config, self.args) + setup_script_intel_classic(build_config, self.args) def setup_intel_llvm(self, build_config: BuildConfig) -> None: ''' @@ -140,7 +140,7 @@ def setup_intel_llvm(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - setup_intel_llvm(build_config, self.args) + setup_script_intel_llvm(build_config, self.args) def setup_nvidia(self, build_config: BuildConfig) -> None: ''' @@ -152,7 +152,7 @@ def setup_nvidia(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - setup_nvidia(build_config, self.args) + setup_script_nvidia(build_config, self.args) def get_path_flags(self, build_config: BuildConfig) -> List[AddFlags]: ''' diff --git a/lfric_build/site_specific/default/setup_cray.py b/lfric_build/site_specific/default/setup_script_cray.py similarity index 97% rename from lfric_build/site_specific/default/setup_cray.py rename to lfric_build/site_specific/default/setup_script_cray.py index 346b452fe..30300557a 100644 --- a/lfric_build/site_specific/default/setup_cray.py +++ b/lfric_build/site_specific/default/setup_script_cray.py @@ -13,7 +13,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository -def setup_cray(build_config: BuildConfig, args: argparse.Namespace) -> None: +def setup_script_cray(build_config: BuildConfig, + args: argparse.Namespace) -> None: # pylint: disable=unused-argument ''' Defines the default flags for ftn. diff --git a/lfric_build/site_specific/default/setup_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py similarity index 97% rename from lfric_build/site_specific/default/setup_gnu.py rename to lfric_build/site_specific/default/setup_script_gnu.py index c00c99b91..aae89e922 100644 --- a/lfric_build/site_specific/default/setup_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -13,7 +13,8 @@ from fab.api import BuildConfig, Category, Linker, ToolRepository -def setup_gnu(build_config: BuildConfig, args: argparse.Namespace) -> None: +def setup_script_gnu(build_config: BuildConfig, + args: argparse.Namespace) -> None: # pylint: disable=unused-argument ''' Defines the default flags for all GNU compilers and linkers. diff --git a/lfric_build/site_specific/default/setup_intel_classic.py b/lfric_build/site_specific/default/setup_script_intel_classic.py similarity index 97% rename from lfric_build/site_specific/default/setup_intel_classic.py rename to lfric_build/site_specific/default/setup_script_intel_classic.py index ab451f30d..83f64019a 100644 --- a/lfric_build/site_specific/default/setup_intel_classic.py +++ b/lfric_build/site_specific/default/setup_script_intel_classic.py @@ -13,8 +13,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository -def setup_intel_classic(build_config: BuildConfig, - args: argparse.Namespace) -> None: +def setup_script_intel_classic(build_config: BuildConfig, + args: argparse.Namespace) -> None: # pylint: disable=unused-argument, too-many-locals ''' Defines the default flags for all Intel classic compilers and linkers. diff --git a/lfric_build/site_specific/default/setup_intel_llvm.py b/lfric_build/site_specific/default/setup_script_intel_llvm.py similarity index 96% rename from lfric_build/site_specific/default/setup_intel_llvm.py rename to lfric_build/site_specific/default/setup_script_intel_llvm.py index d3a99bb68..fe4e38347 100644 --- a/lfric_build/site_specific/default/setup_intel_llvm.py +++ b/lfric_build/site_specific/default/setup_script_intel_llvm.py @@ -13,8 +13,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository -def setup_intel_llvm(build_config: BuildConfig, - args: argparse.Namespace) -> None: +def setup_script_intel_llvm(build_config: BuildConfig, + args: argparse.Namespace) -> None: # pylint: disable=unused-argument, too-many-locals ''' Defines the default flags for all Intel llvm compilers. diff --git a/lfric_build/site_specific/default/setup_nvidia.py b/lfric_build/site_specific/default/setup_script_nvidia.py similarity index 96% rename from lfric_build/site_specific/default/setup_nvidia.py rename to lfric_build/site_specific/default/setup_script_nvidia.py index 63b179a79..83ce3e4d0 100644 --- a/lfric_build/site_specific/default/setup_nvidia.py +++ b/lfric_build/site_specific/default/setup_script_nvidia.py @@ -13,7 +13,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository -def setup_nvidia(build_config: BuildConfig, args: argparse.Namespace) -> None: +def setup_script_nvidia(build_config: BuildConfig, + args: argparse.Namespace) -> None: # pylint: disable=unused-argument ''' Defines the default flags for nvfortran. From a5309c1dbd0ebbc76a62c9a45a5121f3cde4e5db Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 24 Feb 2026 10:48:16 +1100 Subject: [PATCH 32/95] #240 Fixed failing tests, handled warning. --- lfric_build/tests/lfric_base_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index aa839a2d5..f1986a21e 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -185,7 +185,7 @@ def test_get_directory(monkeypatch, tmp_path) -> None: mock_core.mkdir(parents=True) # Create mock LFRic base file location - mock_base_dir = mock_core / "infrastructure" / "build" / "fab" + mock_base_dir = mock_core / "lfric_build" mock_base_dir.mkdir(parents=True) mock_base_file = mock_base_dir / "lfric_base.py" mock_base_file.write_text("", encoding='utf-8') @@ -444,7 +444,8 @@ def test_configurator_step(monkeypatch) -> None: lfric_base = LFRicBase(name="test") monkeypatch.setattr(lfric_base, 'get_rose_meta', mock_meta) - lfric_base.configurator_step() + with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): + lfric_base.configurator_step() # Verify configurator called with correct arguments mock_config.assert_called_once_with( From 9731fa1dbbfdf4ac9faf42720be74e2dbee2e04d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 2 Mar 2026 19:06:59 +1100 Subject: [PATCH 33/95] #240 Remove unit-test as compilation profile. --- lfric_build/site_specific/default/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index a2338bf51..1dc44e3f0 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -44,7 +44,7 @@ def get_valid_profiles(self) -> List[str]: :returns List[str]: list of all supported compiler profiles. ''' - return ["full-debug", "fast-debug", "production", "unit-tests"] + return ["full-debug", "fast-debug", "production"] def update_toolbox(self, build_config: BuildConfig) -> None: ''' From c54edab83aed26b9aa4f3a586f9440c5fd822f16 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 4 Mar 2026 11:31:14 +1100 Subject: [PATCH 34/95] #240 Removed setting up unit-test for gnu (since unit-test was removed). --- lfric_build/site_specific/default/setup_script_gnu.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index aae89e922..ad2bc2e6b 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -78,10 +78,6 @@ def setup_script_gnu(build_config: BuildConfig, # ========== gfortran.add_flags(["-Ofast"], "production") - # unit-tests - # ========== - gfortran.add_flags(runtime + ["-O0"] + init, "unit-tests") - # Set up the linker # ================= # This will implicitly affect all gfortran based linkers, e.g. From 9942f1c563f0ec18bf0be86839c3085a1f2c149f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 6 Mar 2026 18:07:32 +1100 Subject: [PATCH 35/95] #292 Started to add support for pFUNit. --- applications/skeleton/fab_skeleton.py | 16 +-- lfric_build/lfric_base.py | 33 +++-- lfric_build/lfric_base_with_test.py | 196 ++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 19 deletions(-) create mode 100755 lfric_build/lfric_base_with_test.py diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index 63596dab2..9a3f82910 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -22,10 +22,10 @@ # We need to import the base class: sys.path.insert(0, str(Path(__file__).parents[2] / "lfric_build")) -from lfric_base import LFRicBase # noqa: E402 +from lfric_base_with_test import LFRicBaseWithTest # noqa: E402 -class FabSkeleton(LFRicBase): +class FabSkeleton(LFRicBaseWithTest): """ A Fab-based build script for skeleton. It relies on the LFRicBase class to implement the actual functionality, and only provides the required @@ -35,7 +35,9 @@ class FabSkeleton(LFRicBase): """ def __init__(self, name: str = "skeleton") -> None: - super().__init__(name=name) + + apps_dir = Path(__file__).parent + super().__init__(name=name, apps_dir=apps_dir) # Store the root of this apps for later this_file = Path(__file__).resolve() self._this_root = this_file.parent @@ -45,12 +47,8 @@ def grab_files_step(self) -> None: Grabs the required source files and optimisation scripts. """ super().grab_files_step() - dirs = ['applications/skeleton/source/'] - - # pylint: disable=redefined-builtin - for dir in dirs: - grab_folder(self.config, src=self.lfric_core_root / dir, - dst_label='') + grab_folder(self.config, src=self.apps_dir / "source", + dst_label='') # Copy the optimisation scripts into a separate directory grab_folder(self.config, src=self._this_root / "optimisation", diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 3b6e99bbc..6c6bd5b33 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -32,15 +32,20 @@ class LFRicBase(FabBase): :param name: the name to be used for the workspace. Note that the name of the compiler will be added to it. + :param apps_dir: the base directory of the application. :param root_symbol: the symbol (or list of symbols) of the main programs. Defaults to the parameter `name` if not specified. ''' # pylint: disable=too-many-instance-attributes def __init__(self, name: str, + apps_dir: Path, root_symbol: Optional[Union[List[str], str]] = None ): + self._apps_dir = apps_dir + # Will be set to true if a unit-test directory is found + # List of all precision preprocessor symbols and their default. # Used to add corresponding command line options, and then to define # the preprocessor definitions. @@ -65,6 +70,20 @@ def __init__(self, name: str, self._add_python_paths = [str(self.lfric_core_root / "infrastructure" / "build" / "psyclone")] + @property + def apps_dir(self) -> Path: + """ + :returns: the root directory of the application. + """ + return self._apps_dir + + @property + def lfric_core_root(self) -> Path: + ''' + :returns: the root directory of the LFRic core repository. + ''' + return self._lfric_core_root + def define_command_line_options( self, parser: Optional[argparse.ArgumentParser] = None @@ -117,13 +136,6 @@ def __call__(self, parser, namespace, values, option_string=None): return parser - @property - def lfric_core_root(self) -> Path: - ''' - :returns: the root directory of the LFRic core repository. - ''' - return self._lfric_core_root - def setup_site_specific_location(self): ''' This method adds the required directories for site-specific @@ -179,7 +191,7 @@ def define_preprocessor_flags_step(self) -> None: def get_linker_flags(self) -> List[str]: ''' - This method overwrites the base class get_liner_flags. It passes the + This method overwrites the base class get_linker_flags. It passes the libraries that LFRic uses to the linker. Currently, these libraries include yaxt, xios, netcdf and hdf5. @@ -208,8 +220,7 @@ def grab_files_step(self) -> None: dst_label='') # Copy the PSyclone Config file into a separate directory - dir = "etc" - grab_folder(self.config, src=self.lfric_core_root / dir, + grab_folder(self.config, src=self.lfric_core_root / "etc", dst_label='psyclone_config') def find_source_files_step( @@ -228,6 +239,8 @@ def find_source_files_step( self.configurator_step() path_filter_list = list(path_filters) if path_filters else [] + # If testing is used (via LFRicBaseWithTest), unit-test will + # be handled there. path_filter_list.append(Exclude('unit-test', '/test/')) super().find_source_files_step(path_filters=path_filter_list) diff --git a/lfric_build/lfric_base_with_test.py b/lfric_build/lfric_base_with_test.py new file mode 100755 index 000000000..bf4795dba --- /dev/null +++ b/lfric_build/lfric_base_with_test.py @@ -0,0 +1,196 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# Author: J. Henrichs, Bureau of Meteorology +# Author: J. Lyu, Bureau of Meteorology + +""" +This is an OO basic interface to FAB. It allows the typical LFRic +applications to only modify very few settings to have a working FAB build +script. +""" + +import argparse +from pathlib import Path +from typing import List, Optional, Iterable, Union + +from fab.api import (ArtefactSet, Category, Exclude, grab_folder, Include, + input_to_output_fpath, step) + +from lfric_base import LFRicBase + + +class LFRicBaseWithTest(LFRicBase): + ''' + This class adds support for pFUnit based testing. It also adds + a command line option to disable testing. This class will also + automatically detect if there is no unit-test directory and handle + this case correctly. + + :param name: the name to be used for the workspace. Note that + the name of the compiler will be added to it. + :param apps_dir: the base directory of the application. + :param root_symbol: the symbol (or list of symbols) of the main + programs. Defaults to the parameter `name` if not specified. + + ''' + + # The new artefact set to use for pf files + PF_SOURCE = "PF_SOURCE" + + # pylint: disable=too-many-instance-attributes + def __init__(self, name: str, + apps_dir: Path, + root_symbol: Optional[Union[List[str], str]] = None + ): + + self._has_test = False + super().__init__(name, apps_dir=apps_dir, root_symbol=root_symbol) + + def define_command_line_options( + self, + parser: Optional[argparse.ArgumentParser] = None + ) -> argparse.ArgumentParser: + ''' + Adds an option to disable testing + + :param parser: optional a pre-defined argument parser. + + :returns: the argument parser with the LFRic specific options added. + ''' + parser = super().define_command_line_options() + + parser.add_argument( + '--no-test', action="store_true", default=False, + help="Disable compilation of pFUnit tests.") + + return parser + + def get_linker_flags(self) -> List[str]: + ''' + This method adds pFUnit as library if tests are available and enabled. + + :returns: list of flags for the linker. + ''' + libs: list[Path] = [] + if self._has_test: + # TODO: This implies that pfunit will be used when linking the + # actual app. We need improved support for path-specific + # flags in fab to specify a lib only to be used depending + # on output. + libs.append('pfunit') + return libs + super().get_linker_flags() + + def grab_files_step(self) -> None: + ''' + This method adds files from APPS/unit-test if not disabled via the + command line switch. If this directory exists (and testing is not + disabled), it will set `self._has_test` to include compilation of + pFUnit test files in the future build steps. + ''' + + super().grab_files_step() + + unit_test = "unit-test" + # Check if there are unit tests + if (not self.args.no_test and self.apps_dir / unit_test).is_dir(): + grab_folder(self.config, src=self.apps_dir / unit_test, + dst_label=unit_test) + self._has_test = True + + def find_source_files_step( + self, + path_filters: Optional[Iterable[Union[Exclude, Include]]] = None + ) -> None: + ''' + This method overwrites the base class find_source_files_step. + It first calls the configurator_step to set up the configurator. + Then it finds all the source files in the LFRic core directories, + excluding the unit tests. Finally, it calls the templaterator_step. + + :param path_filters: optional list of path filters to be passed to + Fab find_source_files, default is None. + ''' + super().find_source_files_step(path_filters=path_filters) + if not self._has_test: + # Don't do anything else if there are no test files (or testing + # was explicitly disabled on the command line). + return + + self.config.artefact_store[LFRicBaseWithTest.PF_SOURCE] = set() + self.config.artefact_store.copy_artefacts( + ArtefactSet.INITIAL_SOURCE_FILES, + LFRicBaseWithTest.PF_SOURCE, + suffixes=[".pf", ".PF"]) + pfunit = self.config.tool_box.get_tool("pfunit") + driver_f90 = pfunit.get_driver_f90() + # TODO: fab_base needs a `name` property + driver_f90 = driver_f90.replace("program main", + f"program {self._name}_unit_test") + + out_driver = (self.config.build_output / "unit-test" / + f"driver_{self._name}.F90") + out_driver.parent.mkdir(parents=True, exist_ok=True) + with out_driver.open("w", encoding='utf-8') as f: + f.write(driver_f90) + + self.config.artefact_store.add(ArtefactSet.FORTRAN_COMPILER_FILES, + out_driver) + + @step + def preprocess_pfunit_step(self) -> None: + """ + Preprocess all .pf files with pfunit, and create test_list.inc + to list all tests (which is required when preprocessing the + pfunit driver program). + """ + + pf_files = self.config.artefact_store[LFRicBaseWithTest.PF_SOURCE] + pfunit = self.config.tool_box.get_tool("pfunit") + all_tests = [] + for pf_file in pf_files: + all_tests.append(pf_file.stem) + output_fpath = (input_to_output_fpath(config=self.config, + input_path=pf_file) + .with_suffix(".F90")) + output_fpath.parent.mkdir(parents=True, exist_ok=True) + pfunit.process(pf_file, output_fpath) + test_list = self.config.build_output / "unit-test" / "test_list.inc" + with test_list.open("w", encoding="utf-8") as f: + for test_name in all_tests: + f.write(f"ADD_TEST_SUITE({test_name})\n") + + # TODO: That should be path-specific + self.add_preprocessor_flags([f"-D_TEST_SUITES=\"{test_list.name}\"", + "-I", str(pfunit.get_include_path())]) + self._root_symbol.append("skeleton_unit_test") + compiler = self.config.tool_box.get_tool(Category.FORTRAN_COMPILER) + compiler.add_flags(["-I", str(pfunit.get_include_path())]) + + def preprocess_fortran_step(self) -> None: + if self._has_test: + self.preprocess_pfunit_step() + super().preprocess_fortran_step() + + def analyse_step( + self, + ignore_dependencies: Optional[Iterable[str]] = None, + find_programs: bool = False + ) -> None: + ''' + The method overwrites the base class analyse_step. + For LFRic, it first runs the preprocess_x90_step and then runs + psyclone_step. Finally, it calls Fab's analyse for dependency + analysis, ignoring the third party modules that are commonly + used by LFRic. + ''' + if ignore_dependencies is None: + ignore_dependencies = [] + # core/infrastructure/build/import.mk + ignore_dep_list = list(ignore_dependencies) + if self._has_test: + ignore_dep_list += ['pfunit'] + super().analyse_step(ignore_dependencies=ignore_dep_list, + find_programs=find_programs) From 1c7a85fce9a3d7bcb6cbc98fe906c3ff338ee248 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 19 Mar 2026 14:26:36 +1100 Subject: [PATCH 36/95] #240 Updated precision handling as requested in review. --- lfric_build/lfric_base.py | 48 ++++++++-------------------- lfric_build/tests/lfric_base_test.py | 40 +---------------------- 2 files changed, 14 insertions(+), 74 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 3b6e99bbc..efdc707d0 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -43,8 +43,9 @@ def __init__(self, name: str, # List of all precision preprocessor symbols and their default. # Used to add corresponding command line options, and then to define - # the preprocessor definitions. - self._all_precisions = [("RDEF_PRECISION", "64"), + # the preprocessor definitions. Note that precision_other + # becomes RDEF. + self._all_precisions = [("precision_other", "64"), ("R_SOLVER_PRECISION", "32"), ("R_TRAN_PRECISION", "64"), ("R_BL_PRECISION", "64")] @@ -91,29 +92,15 @@ def define_command_line_options( description="Arguments related to setting the floating " "point precision.") - group.add_argument( - '--precision-default', type=str, default=None, - choices=['32', '64'], help="Default precision for reals.") - - # We need to distinguish if a user specified a value (even if it is - # the default), or not. Use the following action for argparse: - class StoreWithFlag(argparse.Action): - """ - Helper class to add a `XX_specified` entry for command line - options that the user has explicitly specified. - """ - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - setattr(namespace, f"{self.dest}_specified", True) - for prec_name, default in self._all_precisions: lower_name = prec_name.lower() + if prec_name == "precision_other": + help_msg = "Precision for other floating point values." + else: + help_msg = f"Precision for '{prec_name}'." group.add_argument( f'--{lower_name}', type=str, choices=['32', '64'], - default=default, action=StoreWithFlag, - help=f"Precision for '{prec_name}'. Default will be " - f"overwritten by ${prec_name} or --precision-default " - f"in this order.") + default=default, help=help_msg) return parser @@ -150,23 +137,14 @@ def define_preprocessor_flags_step(self) -> None: ''' preprocessor_flags: List[str] = [] - # Take the value of --precision-default (or None if not specified): - generic_default = self.args.precision_default - # Check all required precision defines - for prec_name, prec_default in self._all_precisions: + for prec_name, _ in self._all_precisions: # Check if a value was specified on the command line: - if getattr(self.args, f"{prec_name.lower()}_specified", False): - value = getattr(self.args, prec_name.lower()) + value = getattr(self.args, prec_name.lower()) + if prec_name == "precision_other": + preprocessor_flags.append(f"-DRDEF_PRECISION={value}") + else: preprocessor_flags.append(f"-D{prec_name}={value}") - continue - - # No command line option for the current precision name. - # Check if a default was set (--precision-default), otherwise - # use the default for this precision - preprocessor_flags.append( - f"-D{prec_name}=" - f"{generic_default if generic_default else prec_default}") # core/components/lfric-xios/build/import.mk if not self.args.no_xios: diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index f1986a21e..e5a127780 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -227,18 +227,6 @@ def create_frame_info(filename): assert lfric_base.lfric_core_root == mock_core -def test_command_line_options(monkeypatch) -> None: - ''' - Tests LFRic specific command line options. - ''' - monkeypatch.setattr(sys, "argv", ["lfric_base.py", - "--precision-default", "32"]) - - lfric_base = LFRicBase(name="test") - - assert lfric_base.args.precision_default == "32" - - def test_precision_definition_without_default(monkeypatch) -> None: ''' Tests specification of precision if no default precision is @@ -248,7 +236,7 @@ def test_precision_definition_without_default(monkeypatch) -> None: R_*PRECISION default. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py", - "--rdef_precision", "32"]) + "--precision_other", "32"]) monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) lfric_base = LFRicBase(name="test") @@ -265,32 +253,6 @@ def test_precision_definition_without_default(monkeypatch) -> None: assert '-DR_BL_PRECISION=64' in flags -def test_precision_definition_with_default(monkeypatch) -> None: - ''' - Tests specification of precision. Test all ways a precision - can be specified: default command line, explicit command - line, environment variable, and the per R_*PRECISION default. - ''' - monkeypatch.setattr(sys, "argv", ["lfric_base.py", - "--precision-default", "32", - "--rdef_precision", "64"]) - monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) - - lfric_base = LFRicBase(name="test") - lfric_base.define_preprocessor_flags_step() - - flags = lfric_base.preprocess_flags_common - # Explicitly set on command line: - assert '-DRDEF_PRECISION=64' in flags - # Specified default of any precision - assert '-DR_SOLVER_PRECISION=32' in flags - # Specified default of any precision - assert '-DR_TRAN_PRECISION=32' in flags - # Old style environment variables must be ignored, so R_BL_PRECISION - # must still be 32! - assert '-DR_BL_PRECISION=32' in flags - - @pytest.mark.parametrize('no_xios', [True, False]) @pytest.mark.parametrize('mpi', [True, False]) def test_preprocessor_flags(monkeypatch, no_xios, mpi) -> None: From 92008c53be09e84461d15b44498fc0e7b82e2551 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 31 Mar 2026 15:23:23 +1100 Subject: [PATCH 37/95] #292 Changed the base class into a mixin. --- applications/skeleton/fab_skeleton.py | 5 ++-- ...fric_base_with_test.py => pfunit_mixin.py} | 26 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) rename lfric_build/{lfric_base_with_test.py => pfunit_mixin.py} (92%) diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index 9a3f82910..cd4cfc0ae 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -22,10 +22,11 @@ # We need to import the base class: sys.path.insert(0, str(Path(__file__).parents[2] / "lfric_build")) -from lfric_base_with_test import LFRicBaseWithTest # noqa: E402 +from lfric_base import LFRicBase # noqa: E402 +from pfunit_mixin import PfUnitMixin # noqa: E402 -class FabSkeleton(LFRicBaseWithTest): +class FabSkeleton(PfUnitMixin, LFRicBase): """ A Fab-based build script for skeleton. It relies on the LFRicBase class to implement the actual functionality, and only provides the required diff --git a/lfric_build/lfric_base_with_test.py b/lfric_build/pfunit_mixin.py similarity index 92% rename from lfric_build/lfric_base_with_test.py rename to lfric_build/pfunit_mixin.py index bf4795dba..041882d63 100755 --- a/lfric_build/lfric_base_with_test.py +++ b/lfric_build/pfunit_mixin.py @@ -19,12 +19,10 @@ from fab.api import (ArtefactSet, Category, Exclude, grab_folder, Include, input_to_output_fpath, step) -from lfric_base import LFRicBase - -class LFRicBaseWithTest(LFRicBase): +class PfUnitMixin: ''' - This class adds support for pFUnit based testing. It also adds + This mixin adds support for pFUnit based testing. It also adds a command line option to disable testing. This class will also automatically detect if there is no unit-test directory and handle this case correctly. @@ -119,10 +117,10 @@ def find_source_files_step( # was explicitly disabled on the command line). return - self.config.artefact_store[LFRicBaseWithTest.PF_SOURCE] = set() + self.config.artefact_store[PfUnitMixin.PF_SOURCE] = set() self.config.artefact_store.copy_artefacts( ArtefactSet.INITIAL_SOURCE_FILES, - LFRicBaseWithTest.PF_SOURCE, + PfUnitMixin.PF_SOURCE, suffixes=[".pf", ".PF"]) pfunit = self.config.tool_box.get_tool("pfunit") driver_f90 = pfunit.get_driver_f90() @@ -147,7 +145,7 @@ def preprocess_pfunit_step(self) -> None: pfunit driver program). """ - pf_files = self.config.artefact_store[LFRicBaseWithTest.PF_SOURCE] + pf_files = self.config.artefact_store[PfUnitMixin.PF_SOURCE] pfunit = self.config.tool_box.get_tool("pfunit") all_tests = [] for pf_file in pf_files: @@ -170,9 +168,14 @@ def preprocess_pfunit_step(self) -> None: compiler.add_flags(["-I", str(pfunit.get_include_path())]) def preprocess_fortran_step(self) -> None: + """ + Calls Fab's preprocessing of all Fortran files. After preprocessing + the sources, this implementation will then also pre-process the + test files. + """ + super().preprocess_fortran_step() if self._has_test: self.preprocess_pfunit_step() - super().preprocess_fortran_step() def analyse_step( self, @@ -180,11 +183,8 @@ def analyse_step( find_programs: bool = False ) -> None: ''' - The method overwrites the base class analyse_step. - For LFRic, it first runs the preprocess_x90_step and then runs - psyclone_step. Finally, it calls Fab's analyse for dependency - analysis, ignoring the third party modules that are commonly - used by LFRic. + The method overwrites the base class analyse_step and adds + pfunit to be ignored in the analysis step. ''' if ignore_dependencies is None: ignore_dependencies = [] From e0984ee3ca3cb7e45cc0b2bb10968a4a1157028a Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Apr 2026 15:28:25 +1100 Subject: [PATCH 38/95] #292 Fixed small bugs and incorrect name of test symbol. --- lfric_build/pfunit_mixin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index 041882d63..10a0033ba 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -93,7 +93,7 @@ def grab_files_step(self) -> None: unit_test = "unit-test" # Check if there are unit tests - if (not self.args.no_test and self.apps_dir / unit_test).is_dir(): + if (not self.args.no_test) and (self.apps_dir / unit_test).is_dir(): grab_folder(self.config, src=self.apps_dir / unit_test, dst_label=unit_test) self._has_test = True @@ -163,7 +163,8 @@ def preprocess_pfunit_step(self) -> None: # TODO: That should be path-specific self.add_preprocessor_flags([f"-D_TEST_SUITES=\"{test_list.name}\"", "-I", str(pfunit.get_include_path())]) - self._root_symbol.append("skeleton_unit_test") + # TODO: fab_base needs a `name` property + self._root_symbol.append(f"{self._name}_unit_test") compiler = self.config.tool_box.get_tool(Category.FORTRAN_COMPILER) compiler.add_flags(["-I", str(pfunit.get_include_path())]) @@ -173,9 +174,12 @@ def preprocess_fortran_step(self) -> None: the sources, this implementation will then also pre-process the test files. """ - super().preprocess_fortran_step() + + # We need to call preprocess_pfunit first, since it will create + # the test_list.inc file if self._has_test: self.preprocess_pfunit_step() + super().preprocess_fortran_step() def analyse_step( self, From f6185d780172047e83a46e9919cb117cad2d76c4 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Apr 2026 15:31:40 +1100 Subject: [PATCH 39/95] #292 Ignore fab-workspace directories. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 164a542f5..506673052 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ __pycache__ # LFRic CL Builds applications/**/bin applications/**/working +applications/**/fab-workspace applications/**/test applications/**/documents applications/**/example*/ From afd5c126eb2bd9005a202719e2a535bd7152c8fa Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Apr 2026 15:34:02 +1100 Subject: [PATCH 40/95] #292 Added pfunit tool. --- lfric_build/pfunit.py | 69 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 lfric_build/pfunit.py diff --git a/lfric_build/pfunit.py b/lfric_build/pfunit.py new file mode 100644 index 000000000..d11705ea7 --- /dev/null +++ b/lfric_build/pfunit.py @@ -0,0 +1,69 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +"""This file contains the Rsync class for synchronising file trees. +""" + +import logging +import os +from pathlib import Path + +from fab.tools.tool import Tool + + +logger = logging.getLogger(__name__) + + +class PfUnit(Tool): + """ + This is a class to encapsulate pFUnit. It relies on the environment + variable $PFUNIT to indicate the location of the source code. + This is required since besides .mod files and executable, it also + contains the source code for a Fortran driver program . + It assumes that pFUnit's preprocessor `funitproc` is in $PFUNIT/bin. + """ + + def __init__(self): + pfunit_home = os.environ.get("PFUNIT", "") + if not pfunit_home: + logger.error("$PFUNIT not defined in an environment, testing will" + "likely not work.") + self._pfunit_home = Path(pfunit_home) + + exec_name = self._pfunit_home / "bin" / "funitproc" + super().__init__("funitproc", exec_name=exec_name, category="pfunit", + availability_option="-v") + + def get_root_path(self) -> Path: + """ + :returns: the root path of pFUnit. + """ + return self._pfunit_home + + def get_include_path(self) -> Path: + """ + :returns: the include directory for PFUnit. + """ + return self._pfunit_home / "include" + + def get_driver_f90(self) -> str: + """ + :returns: the content of pFUnit's driver.F90 file. + """ + driver_path = self._pfunit_home / "include" / "driver.F90" + with driver_path.open("r", encoding='utf-8') as f: + driver_f90 = f.read() + return driver_f90 + + def process(self, pf_path: Path, + f90_out_path: Path): + """ + Processes the .pf file to create an output f90 file. + + :param pf_path: the input path. + :param f90_out_path: destination path. + """ + return self.run(additional_parameters=[pf_path, f90_out_path]) From b707e154e80162594e6ec488a29d28d29612a7c9 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Apr 2026 15:49:13 +1100 Subject: [PATCH 41/95] #292 Updated documentation. --- lfric_build/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lfric_build/README.md b/lfric_build/README.md index 5c22299be..2270fb957 100644 --- a/lfric_build/README.md +++ b/lfric_build/README.md @@ -52,3 +52,18 @@ that the available compilation profiles can vary from site to site (see If Fab has issues finding a compiler, you can use the Fab debug option ```--available-compilers```, which will list all compilers and linkers Fab has identified as being available. + +## PFUnit testin +Any application script can additionally inherit from the ``pfunit_mixin.py`` +mixin, e.g.: +``` +from lfric_base import LFRicBase # noqa: E402 +from pfunit_mixin import PfUnitMixin # noqa: E402 + + +class FabSkeleton(PfUnitMixin, LFRicBase): + +``` +This will by default also build any tests in a ``unit-test`` directory. +Additionally, a new command line option ``--no-test`` is added if +building of the tests should be disabled. From 835ab8bbb120396909c6b75541eb162816bfdf02 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Apr 2026 16:06:46 +1100 Subject: [PATCH 42/95] #292 Fixed typo. --- lfric_build/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lfric_build/README.md b/lfric_build/README.md index 2270fb957..8cd8d14ea 100644 --- a/lfric_build/README.md +++ b/lfric_build/README.md @@ -53,7 +53,8 @@ If Fab has issues finding a compiler, you can use the Fab debug option ```--available-compilers```, which will list all compilers and linkers Fab has identified as being available. -## PFUnit testin +## Testing with pFUnit + Any application script can additionally inherit from the ``pfunit_mixin.py`` mixin, e.g.: ``` From 280268bd6b57499a4bad5140ff501fdb5c00dfeb Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 9 Apr 2026 16:32:49 +1000 Subject: [PATCH 43/95] #240 Addressed issues raised in review. --- lfric_build/README.md | 37 ++++++++++++++----- .../site_specific/default/setup_script_gnu.py | 2 +- .../site_specific/nci_gadi/__init__.py | 0 3 files changed, 28 insertions(+), 11 deletions(-) delete mode 100644 lfric_build/site_specific/nci_gadi/__init__.py diff --git a/lfric_build/README.md b/lfric_build/README.md index 5c22299be..55518f596 100644 --- a/lfric_build/README.md +++ b/lfric_build/README.md @@ -5,8 +5,8 @@ LFRic core requirements of course). ## Setting up Site- and Platform-specific Settings Site- and platform-specific settings are contained in -```$LFRIC_CORE/infrastructure/build/site-specific/${SITE}-${PLATFORM}``` -The default settings are in ```.../site-specific/default``` (and at this +```$LFRIC_CORE/infrastructure/build/site_specific/${SITE}-${PLATFORM}``` +The default settings are in ```site_specific/default``` (and at this stage each other site-specific setup inherits the values set in the default, and then adds or modifies settings). The Fab build system provides various callbacks to the ```config.py``` file in the corresponding directory (details @@ -31,14 +31,31 @@ In order to build the skeleton apps, change into the directory and use the following command: ``` -./fab_skeleton.py --nprocs 4 --site nci --platform gadi --suite intel-classic +./fab_skeleton.py --nprocs 4 --suite gnu ``` -Select an appropriate number of processes to run in parallel, and your site and platform. -If you don't have a default compiler suite in your site-specific setup (or -want to use a non-default suite), use the ``--suite`` option. Once the process is finished, -you should have a binary in the directory -```./fab-workspace/skeleton-full-debug-COMPILER``` (where ```COMPILER``` is the compiler -used, e.g. ```mpif90-gfortran```). +Select an appropriate number of processes to run in parallel, and a compiler +suite, e.g. one of ```gnu```, ```cray```, ```intel-classic```, ```intel-llvm``` +or ```nvidia```. Once the process is finished, you should have a binary in the +directory ```./fab-workspace/skeleton-full-debug-COMPILER``` (where +```COMPILER``` is the compiler used, e.g. ```mpif90-gfortran```). + +Likely, you will have to setup corresponding compiler and linker options, e.g. +include paths, library paths and libraries to link. The directory ```site_specific``` +contains site-specific setup files, and one called ```default``` (which is +used in the above example if no site is selected). If your site does not exist, +create a corresponding directory for your site and platform in the format +```site_platform``` by using an existing site as template. See also the +[Fab documentation](https://metoffice.github.io/fab/fab_base/config.html) +for details about specifying compiler and linker options). +You can then use the command line options ```--site``` and ```--platform``` +to pick your setup, e.g.: + +``` +./fab_skeleton.py --nprocs 4 --site nci --platform gadi --suite intel-llvm +``` + +This would use the file ```site_specific/nci_gadi/config.py```, and all additional +compiler and linker options defined there. Using ```./fab_skeleton.py -h``` will show a help message with all supported command line options (and their default value). If a default value is listed using an environment @@ -49,6 +66,6 @@ A different compilation profile can be specified using ```--profile``` option. N that the available compilation profiles can vary from site to site (see [Fab documentation](https://metoffice.github.io/fab/fab_base/config.html) for details). -If Fab has issues finding a compiler, you can use the Fab debug option +If Fab has issues finding a compiler, you can use the Fab debug command line option ```--available-compilers```, which will list all compilers and linkers Fab has identified as being available. diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index ad2bc2e6b..7e77204e5 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -33,7 +33,7 @@ def setup_script_gnu(build_config: BuildConfig, return if gfortran.get_version() < (4, 9): - raise RuntimeError(f"GFortran is too old to build dynamo. " + raise RuntimeError(f"GFortran is too old to build LFRic. " f"Must be at least 4.9.0, it is " f"'{gfortran.get_version_string()}'.") diff --git a/lfric_build/site_specific/nci_gadi/__init__.py b/lfric_build/site_specific/nci_gadi/__init__.py deleted file mode 100644 index e69de29bb..000000000 From ecdff1d8804bab8a3777699bbd5ca6b85a11fd90 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 9 Apr 2026 16:34:37 +1000 Subject: [PATCH 44/95] #240 Check that NetCDF is available before starting to compile. --- lfric_build/lfric_base.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index efdc707d0..315fc29e5 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -17,9 +17,9 @@ import sys from typing import List, Optional, Iterable, Union -from fab.api import (ArtefactSet, BuildConfig, Exclude, grab_folder, Include, - input_to_output_fpath, preprocess_x90, psyclone, step, - SuffixFilter) +from fab.api import (ArtefactSet, BuildConfig, Category, Exclude, grab_folder, + Include, input_to_output_fpath, preprocess_x90, psyclone, + step, SuffixFilter) from fab.fab_base.fab_base import FabBase from configurator import configurator @@ -65,6 +65,16 @@ def __init__(self, name: str, # paths might need to be added later. self._add_python_paths = [str(self.lfric_core_root / "infrastructure" / "build" / "psyclone")] + linker = self.config.tool_box.get_tool(Category.LINKER, + mpi=self.config.mpi, + openmp=self.config.openmp, + enforce_fortran_linker=True) + try: + linker.get_lib_flags("netcdf") + except RuntimeError as err: + msg = (f"LFRic needs NetCDF, but the linker '{linker.name}' " + f"has no NetCDF library setting defined. Aborting.") + raise RuntimeError(msg) from err def define_command_line_options( self, From 2b7d2ae4a26613e327a43bd47c8f62a69a195150 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 23 Apr 2026 21:14:34 +1000 Subject: [PATCH 45/95] #292 Use new Category implementation. --- lfric_build/pfunit.py | 6 +++++- lfric_build/pfunit_mixin.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lfric_build/pfunit.py b/lfric_build/pfunit.py index d11705ea7..32e6d6b5a 100644 --- a/lfric_build/pfunit.py +++ b/lfric_build/pfunit.py @@ -12,6 +12,7 @@ from pathlib import Path from fab.tools.tool import Tool +from fab.tools.category import Category logger = logging.getLogger(__name__) @@ -33,8 +34,11 @@ def __init__(self): "likely not work.") self._pfunit_home = Path(pfunit_home) + # Create a new category of pFUnit + Category("PFUNIT") exec_name = self._pfunit_home / "bin" / "funitproc" - super().__init__("funitproc", exec_name=exec_name, category="pfunit", + super().__init__("funitproc", exec_name=exec_name, + category=Category.PFUNIT, availability_option="-v") def get_root_path(self) -> Path: diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index 10a0033ba..b4c382a37 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -122,7 +122,8 @@ def find_source_files_step( ArtefactSet.INITIAL_SOURCE_FILES, PfUnitMixin.PF_SOURCE, suffixes=[".pf", ".PF"]) - pfunit = self.config.tool_box.get_tool("pfunit") + + pfunit = self.config.tool_box.get_tool(Category.PFUNIT) driver_f90 = pfunit.get_driver_f90() # TODO: fab_base needs a `name` property driver_f90 = driver_f90.replace("program main", @@ -146,7 +147,7 @@ def preprocess_pfunit_step(self) -> None: """ pf_files = self.config.artefact_store[PfUnitMixin.PF_SOURCE] - pfunit = self.config.tool_box.get_tool("pfunit") + pfunit = self.config.tool_box.get_tool(Category.PFUNIT) all_tests = [] for pf_file in pf_files: all_tests.append(pf_file.stem) From ed7203e9e0065ba87a658961ef71bb7f7a3482a6 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 4 May 2026 11:03:12 +1000 Subject: [PATCH 46/95] Update lfric_build/site_specific/default/setup_script_gnu.py Co-authored-by: Sam Clarke-Green <74185251+t00sa@users.noreply.github.com> --- lfric_build/site_specific/default/setup_script_gnu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index 7e77204e5..a82f295ae 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -10,7 +10,7 @@ import argparse from typing import cast -from fab.api import BuildConfig, Category, Linker, ToolRepository +from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository def setup_script_gnu(build_config: BuildConfig, From 6ea9af2a6445e26e1a11d53d420a2b35c06cd873 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 4 May 2026 11:03:44 +1000 Subject: [PATCH 47/95] Update lfric_build/site_specific/default/setup_script_gnu.py Co-authored-by: Sam Clarke-Green <74185251+t00sa@users.noreply.github.com> --- lfric_build/site_specific/default/setup_script_gnu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index a82f295ae..d20ba566b 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -31,6 +31,7 @@ def setup_script_gnu(build_config: BuildConfig, gfortran = tr.get_tool(Category.FORTRAN_COMPILER, "mpif90-gfortran") if not gfortran.is_available: return + gfortran = cast(Compiler, gfortran) if gfortran.get_version() < (4, 9): raise RuntimeError(f"GFortran is too old to build LFRic. " From 513da0b3639bd6788d87bab108751c086b53d4c1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 4 May 2026 11:04:10 +1000 Subject: [PATCH 48/95] Update lfric_build/site_specific/default/config.py Co-authored-by: Sam Clarke-Green <74185251+t00sa@users.noreply.github.com> --- lfric_build/site_specific/default/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 1dc44e3f0..63d9590f4 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -24,8 +24,8 @@ class Config: scripts to allow site-specific customisations. ''' - def __init__(self): - self._args = None + def __init__(self) -> None: + self._args: argparse.Namespace @property def args(self) -> argparse.Namespace: From 7bf6c2a073393a084cb31593887f4b74958ad9a6 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 4 May 2026 11:04:34 +1000 Subject: [PATCH 49/95] Update lfric_build/site_specific/default/config.py Co-authored-by: Sam Clarke-Green <74185251+t00sa@users.noreply.github.com> --- lfric_build/site_specific/default/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 63d9590f4..ce1ccf423 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -66,7 +66,7 @@ def update_toolbox(self, build_config: BuildConfig) -> None: # compilation flags. This 'base' is not accessible to # the user, so it's not part of the profile list. Also, # make it inherit from the default profile '', so that - # a user does not have to specify the "base" profile. + # a user does not have to specify the 'base' profile. # Note that we set this even if a compiler is not available. # This is required in case that compilers are not in PATH, # so e.g. mpif90-ifort works, but ifort cannot be found. From 101ea01bda505b01dc4f17932c593496dfb29046 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 4 May 2026 11:55:23 +1000 Subject: [PATCH 50/95] #240 Added copyright statement. --- lfric_build/site_specific/default/config.py | 9 +++++++-- lfric_build/site_specific/default/setup_script_cray.py | 6 ++++++ lfric_build/site_specific/default/setup_script_gnu.py | 6 ++++++ .../site_specific/default/setup_script_intel_classic.py | 6 ++++++ .../site_specific/default/setup_script_intel_llvm.py | 6 ++++++ lfric_build/site_specific/default/setup_script_nvidia.py | 6 ++++++ lfric_build/site_specific/meto_ex1a/config.py | 6 ++++++ lfric_build/site_specific/ncas_ex/config.py | 6 ++++++ lfric_build/site_specific/nci_gadi/config.py | 6 ++++++ lfric_build/site_specific/niwa_xc50/config.py | 6 ++++++ 10 files changed, 61 insertions(+), 2 deletions(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index ce1ccf423..08a5b3e8d 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -1,8 +1,13 @@ #! /usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## ''' -This module contains the default Baf configuration class. +This module contains the default Fab configuration class. ''' import argparse @@ -19,7 +24,7 @@ class Config: ''' - This class is the default Configuration object for Baf builds. + This class is the default Configuration object for Fab builds. It provides several callbacks which will be called from the build scripts to allow site-specific customisations. ''' diff --git a/lfric_build/site_specific/default/setup_script_cray.py b/lfric_build/site_specific/default/setup_script_cray.py index 30300557a..4e38d2b93 100644 --- a/lfric_build/site_specific/default/setup_script_cray.py +++ b/lfric_build/site_specific/default/setup_script_cray.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This file contains a function that sets the default flags for the Cray compilers and linkers in the ToolRepository. diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index d20ba566b..ee0eac60b 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This file contains a function that sets the default flags for all GNU based compilers and linkers in the ToolRepository. diff --git a/lfric_build/site_specific/default/setup_script_intel_classic.py b/lfric_build/site_specific/default/setup_script_intel_classic.py index 83f64019a..21b20ff87 100644 --- a/lfric_build/site_specific/default/setup_script_intel_classic.py +++ b/lfric_build/site_specific/default/setup_script_intel_classic.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This file contains a function that sets the default flags for all Intel classic based compilers in the ToolRepository (ifort, icc). diff --git a/lfric_build/site_specific/default/setup_script_intel_llvm.py b/lfric_build/site_specific/default/setup_script_intel_llvm.py index fe4e38347..7c8737ac5 100644 --- a/lfric_build/site_specific/default/setup_script_intel_llvm.py +++ b/lfric_build/site_specific/default/setup_script_intel_llvm.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This file contains a function that sets the default flags for all Intel llvm based compilers and linkers in the ToolRepository (ifx, icx). diff --git a/lfric_build/site_specific/default/setup_script_nvidia.py b/lfric_build/site_specific/default/setup_script_nvidia.py index 83ce3e4d0..5f7677c38 100644 --- a/lfric_build/site_specific/default/setup_script_nvidia.py +++ b/lfric_build/site_specific/default/setup_script_nvidia.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This file contains a function that sets the default flags for the NVIDIA compilers and linkers in the ToolRepository. diff --git a/lfric_build/site_specific/meto_ex1a/config.py b/lfric_build/site_specific/meto_ex1a/config.py index c62925243..65af2d5cb 100644 --- a/lfric_build/site_specific/meto_ex1a/config.py +++ b/lfric_build/site_specific/meto_ex1a/config.py @@ -1,5 +1,11 @@ #! /usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + '''This module contains a setup for METO-EX1A ''' diff --git a/lfric_build/site_specific/ncas_ex/config.py b/lfric_build/site_specific/ncas_ex/config.py index 7d287e5ee..122af3dce 100644 --- a/lfric_build/site_specific/ncas_ex/config.py +++ b/lfric_build/site_specific/ncas_ex/config.py @@ -1,5 +1,11 @@ #! /usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + '''This module contains a setup for NCAS-EX (archer2) ''' diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 20d015b9a..3d7b22e01 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -1,5 +1,11 @@ #! /usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This module contains the default configuration for NCI. It will be invoked by the Baf scripts. This script: diff --git a/lfric_build/site_specific/niwa_xc50/config.py b/lfric_build/site_specific/niwa_xc50/config.py index f9e6008b2..42facb429 100644 --- a/lfric_build/site_specific/niwa_xc50/config.py +++ b/lfric_build/site_specific/niwa_xc50/config.py @@ -1,5 +1,11 @@ #! /usr/bin/env python3 +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + ''' This module contains a setup NIWA's XC-50 ''' From 97490d75e6618ecf505285f102c831baae80ddd9 Mon Sep 17 00:00:00 2001 From: Sam Clarke-Green Date: Tue, 5 May 2026 10:38:45 +0100 Subject: [PATCH 51/95] Add fab-workspace to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 164a542f5..64f3e8490 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ applications/**/working applications/**/test applications/**/documents applications/**/example*/ +applications/**/fab-workspace/ mesh_tools/**/bin mesh_tools/**/working mesh_tools/**/test From 3f9cc9e04ce0df2ee0277d0ddabfa0d0ee981c49 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 10:36:11 +1000 Subject: [PATCH 52/95] #292 Removed old-style typing. --- lfric_build/pfunit.py | 73 ------------------- lfric_build/site_specific/default/config.py | 8 +- .../default/setup_script_cray.py | 4 + .../site_specific/default/setup_script_gnu.py | 4 + .../default/setup_script_intel_classic.py | 4 + .../default/setup_script_intel_llvm.py | 4 + .../default/setup_script_nvidia.py | 4 + lfric_build/site_specific/meto_ex1a/config.py | 4 + lfric_build/site_specific/ncas_ex/config.py | 4 + 9 files changed, 29 insertions(+), 80 deletions(-) delete mode 100644 lfric_build/pfunit.py diff --git a/lfric_build/pfunit.py b/lfric_build/pfunit.py deleted file mode 100644 index 32e6d6b5a..000000000 --- a/lfric_build/pfunit.py +++ /dev/null @@ -1,73 +0,0 @@ -############################################################################## -# (c) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT -# which you should have received as part of this distribution -############################################################################## - -"""This file contains the Rsync class for synchronising file trees. -""" - -import logging -import os -from pathlib import Path - -from fab.tools.tool import Tool -from fab.tools.category import Category - - -logger = logging.getLogger(__name__) - - -class PfUnit(Tool): - """ - This is a class to encapsulate pFUnit. It relies on the environment - variable $PFUNIT to indicate the location of the source code. - This is required since besides .mod files and executable, it also - contains the source code for a Fortran driver program . - It assumes that pFUnit's preprocessor `funitproc` is in $PFUNIT/bin. - """ - - def __init__(self): - pfunit_home = os.environ.get("PFUNIT", "") - if not pfunit_home: - logger.error("$PFUNIT not defined in an environment, testing will" - "likely not work.") - self._pfunit_home = Path(pfunit_home) - - # Create a new category of pFUnit - Category("PFUNIT") - exec_name = self._pfunit_home / "bin" / "funitproc" - super().__init__("funitproc", exec_name=exec_name, - category=Category.PFUNIT, - availability_option="-v") - - def get_root_path(self) -> Path: - """ - :returns: the root path of pFUnit. - """ - return self._pfunit_home - - def get_include_path(self) -> Path: - """ - :returns: the include directory for PFUnit. - """ - return self._pfunit_home / "include" - - def get_driver_f90(self) -> str: - """ - :returns: the content of pFUnit's driver.F90 file. - """ - driver_path = self._pfunit_home / "include" / "driver.F90" - with driver_path.open("r", encoding='utf-8') as f: - driver_f90 = f.read() - return driver_f90 - - def process(self, pf_path: Path, - f90_out_path: Path): - """ - Processes the .pf file to create an output f90 file. - - :param pf_path: the input path. - :param f90_out_path: destination path. - """ - return self.run(additional_parameters=[pf_path, f90_out_path]) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 08a5b3e8d..57daf31f2 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -92,8 +92,7 @@ def handle_command_line_options(self, args: argparse.Namespace) -> None: options have been added. This is for example used to add Vernier profiling flags, which are site-specific. - :param argparse.Namespace args: the command line options added in - the site configs + :param args: the command line options added in the site configs ''' # Keep a copy of the args, so they can be used when # initialising compilers @@ -107,7 +106,6 @@ def setup_cray(self, build_config: BuildConfig) -> None: compiler modes). :param build_config: the Fab build configuration instance - :type build_config: :py:class:`fab.BuildConfig` ''' setup_script_cray(build_config, self.args) @@ -119,7 +117,6 @@ def setup_gnu(self, build_config: BuildConfig) -> None: compiler modes). :param build_config: the Fab build configuration instance - :type build_config: :py:class:`fab.BuildConfig` ''' setup_script_gnu(build_config, self.args) @@ -131,7 +128,6 @@ def setup_intel_classic(self, build_config: BuildConfig) -> None: compiler modes). :param build_config: the Fab build configuration instance - :type build_config: :py:class:`fab.BuildConfig` ''' setup_script_intel_classic(build_config, self.args) @@ -143,7 +139,6 @@ def setup_intel_llvm(self, build_config: BuildConfig) -> None: compiler modes). :param build_config: the Fab build configuration instance - :type build_config: :py:class:`fab.BuildConfig` ''' setup_script_intel_llvm(build_config, self.args) @@ -155,7 +150,6 @@ def setup_nvidia(self, build_config: BuildConfig) -> None: compiler modes). :param build_config: the Fab build configuration instance - :type build_config: :py:class:`fab.BuildConfig` ''' setup_script_nvidia(build_config, self.args) diff --git a/lfric_build/site_specific/default/setup_script_cray.py b/lfric_build/site_specific/default/setup_script_cray.py index 4e38d2b93..83c17771e 100644 --- a/lfric_build/site_specific/default/setup_script_cray.py +++ b/lfric_build/site_specific/default/setup_script_cray.py @@ -118,6 +118,10 @@ def setup_script_cray(build_config: BuildConfig, linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) + linker.add_post_lib_flags(["-lcraystdc++"]) # Using the GNU compiler on Crays for now needs the additional diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index ee0eac60b..ff4c91d08 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -113,5 +113,9 @@ def setup_script_gnu(build_config: BuildConfig, linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) + # Always link with C++ libs linker.add_post_lib_flags(["-lstdc++"], "base") diff --git a/lfric_build/site_specific/default/setup_script_intel_classic.py b/lfric_build/site_specific/default/setup_script_intel_classic.py index 21b20ff87..6995638d9 100644 --- a/lfric_build/site_specific/default/setup_script_intel_classic.py +++ b/lfric_build/site_specific/default/setup_script_intel_classic.py @@ -112,5 +112,9 @@ def setup_script_intel_classic(build_config: BuildConfig, linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) + # Always link with C++ libs linker.add_post_lib_flags(["-lstdc++"]) diff --git a/lfric_build/site_specific/default/setup_script_intel_llvm.py b/lfric_build/site_specific/default/setup_script_intel_llvm.py index 7c8737ac5..bc430283b 100644 --- a/lfric_build/site_specific/default/setup_script_intel_llvm.py +++ b/lfric_build/site_specific/default/setup_script_intel_llvm.py @@ -94,5 +94,9 @@ def setup_script_intel_llvm(build_config: BuildConfig, linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) + # Always link with C++ libs linker.add_post_lib_flags(["-lstdc++"]) diff --git a/lfric_build/site_specific/default/setup_script_nvidia.py b/lfric_build/site_specific/default/setup_script_nvidia.py index 5f7677c38..434b84078 100644 --- a/lfric_build/site_specific/default/setup_script_nvidia.py +++ b/lfric_build/site_specific/default/setup_script_nvidia.py @@ -113,5 +113,9 @@ def setup_script_nvidia(build_config: BuildConfig, linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) + # Always link with C++ libs linker.add_post_lib_flags(lib_flags) diff --git a/lfric_build/site_specific/meto_ex1a/config.py b/lfric_build/site_specific/meto_ex1a/config.py index 65af2d5cb..2bd74e173 100644 --- a/lfric_build/site_specific/meto_ex1a/config.py +++ b/lfric_build/site_specific/meto_ex1a/config.py @@ -46,3 +46,7 @@ def setup_cray(self, build_config: BuildConfig): # these flags for now until the transition to pkg-config linker.add_lib_flags("netcdf", ["-lnetcdff", "-lnetcdf", "-lnetcdf", "-lm"]) + + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) diff --git a/lfric_build/site_specific/ncas_ex/config.py b/lfric_build/site_specific/ncas_ex/config.py index 122af3dce..849517fb9 100644 --- a/lfric_build/site_specific/ncas_ex/config.py +++ b/lfric_build/site_specific/ncas_ex/config.py @@ -56,3 +56,7 @@ def setup_cray(self, build_config: BuildConfig): linker.add_lib_flags("shumlib", ["-lshum"]) linker.add_lib_flags("vernier", ["-lvernier_f", "-lvernier_c", "-lvernier"]) + + # This likely needs adjusting, pfunit required fargparse and gftl + linker.add_lib_flags("pfunit", ["-lfunit", "-lpfunit", + "-lfargparse", "lgftl-shared-v2"]) From 6682d8754580992c54a5f90456d2d9e55922fc16 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 10:36:24 +1000 Subject: [PATCH 53/95] #292 Updated nci gadi config. --- lfric_build/site_specific/nci_gadi/config.py | 83 +++++++++++++++----- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 3d7b22e01..b5d60fa6c 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -8,42 +8,89 @@ ''' This module contains the default configuration for NCI. It will be invoked -by the Baf scripts. This script: -- sets intel-classic as the default compiler suite to use. -- Adds the tau compiler wrapper as (optional) compilers to the ToolRepository. +by the Fab scripts. This script sets intel-llvm as the default compiler +suite to use, and adds the required site-specific linker and include flag. ''' from fab.api import Category, ToolRepository +from fab.tools.pfunit import PfUnit from default.config import Config as DefaultConfig class Config(DefaultConfig): ''' - For NCI, make intel the default, and add the Tau wrapper. + For NCI, make intel the default, and setup link paths for gnu, + intel-classic and intel-llvm. ''' def __init__(self): super().__init__() tr = ToolRepository() - tr.set_default_compiler_suite("intel-classic") + tr.set_default_compiler_suite("intel-llvm") - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): + def setup_gnu(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Gnu compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + ''' + super().setup_gnu(build_config) + linker = tr.get_tool(Category.LINKER, "linker-gnu") + # Add netcdf and pfunit flags + self._setup_linker(linker) + + def setup_intel_classic(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Gnu compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + ''' + super().setup_intel_classic(build_config) + linker = tr.get_tool(Category.LINKER, "linker-intel-classic") + # Add netcdf and pfunit flags + self._setup_linker(linker) + + def setup_intel_llvm(self, build_config: BuildConfig) -> None: + ''' + This method sets up the Gnu compiler and linker flags. + For now call an external function, since it is expected that + this configuration can be very lengthy (once we support + compiler modes). + + :param build_config: the Fab build configuration instance + ''' + super().setup_intel_llvm(build_config) + linker = tr.get_tool(Category.LINKER, "linker-intel-llvm") + # Add netcdf and pfunit flags + self._setup_linker(linker) + + def _setup_linker(self, linker: Linker) -> None: + """ + Generic setup of a linker. This adds netcdf and pfunit definitions. + + :param linker: the linker instance to setup + """ + tr = ToolRepository() shell = tr.get_default(Category.SHELL) # We must remove the trailing new line, and create a list: nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], capture_output=True).strip().split() - linker = tr.get_tool(Category.LINKER, "linker-tau-ifort") + linker.add_lib_flags("netcdf", nc_flibs, silent_replace=True) - # Setup all linker flags: - linker.add_lib_flags("netcdf", nc_flibs) - linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) - linker.add_lib_flags("xios", ["-lxios"]) - linker.add_lib_flags("hdf5", ["-lhdf5"]) - linker.add_lib_flags("shumlib", ["-lshum"]) + pfunit = tr.get_tool(Category.PFUNIT, "funitproc") + pfunit_root = pfunit.get_root_path() + spack_view = os.environ.get("SPACK_ENV_VIEW", "") - # Always link with C++ libs - linker.add_post_lib_flags(["-lstdc++"]) + linker.add_lib_flags( + "pfunit", + [f"-L{pfunit_root}/lib", "-lfunit", "-lpfunit", + f"-L{spack_view}/FARGPARSE-1.7/lib/", "-lfargparse", + f"-L{spack_view}/GFTL_SHARED-1.8/lib", "-lgftl-shared-v2", + ], silent_replace=True) From acc5c8e4771f89c994f7c8ecb0f0206fa55f63a7 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 10:45:05 +1000 Subject: [PATCH 54/95] #292 Updated NCI setup. --- lfric_build/site_specific/nci_gadi/config.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index b5d60fa6c..7561e7b56 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -12,7 +12,9 @@ suite to use, and adds the required site-specific linker and include flag. ''' -from fab.api import Category, ToolRepository +import os + +from fab.api import BuildConfig, Category, Linker, ToolRepository from fab.tools.pfunit import PfUnit from default.config import Config as DefaultConfig @@ -39,7 +41,8 @@ def setup_gnu(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' super().setup_gnu(build_config) - linker = tr.get_tool(Category.LINKER, "linker-gnu") + tr = ToolRepository() + linker = tr.get_tool(Category.LINKER, "linker-gfortran") # Add netcdf and pfunit flags self._setup_linker(linker) @@ -53,7 +56,8 @@ def setup_intel_classic(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' super().setup_intel_classic(build_config) - linker = tr.get_tool(Category.LINKER, "linker-intel-classic") + tr = ToolRepository() + linker = tr.get_tool(Category.LINKER, "linker-ifort") # Add netcdf and pfunit flags self._setup_linker(linker) @@ -67,13 +71,16 @@ def setup_intel_llvm(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' super().setup_intel_llvm(build_config) - linker = tr.get_tool(Category.LINKER, "linker-intel-llvm") + tr = ToolRepository() + linker = tr.get_tool(Category.LINKER, "linker-ifx") # Add netcdf and pfunit flags self._setup_linker(linker) def _setup_linker(self, linker: Linker) -> None: """ - Generic setup of a linker. This adds netcdf and pfunit definitions. + Generic setup of a linker (since in the NCI container environments the paths + are actually the same, independent of the compiler used). This adds netcdf and + pfunit definitions. :param linker: the linker instance to setup """ From c7e03ee31c849a84a4ea60ca3631a94642269590 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 10:50:34 +1000 Subject: [PATCH 55/95] #292 Fixed coding style. --- lfric_build/site_specific/nci_gadi/config.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 7561e7b56..2338085a1 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -15,7 +15,6 @@ import os from fab.api import BuildConfig, Category, Linker, ToolRepository -from fab.tools.pfunit import PfUnit from default.config import Config as DefaultConfig @@ -78,9 +77,9 @@ def setup_intel_llvm(self, build_config: BuildConfig) -> None: def _setup_linker(self, linker: Linker) -> None: """ - Generic setup of a linker (since in the NCI container environments the paths - are actually the same, independent of the compiler used). This adds netcdf and - pfunit definitions. + Generic setup of a linker (since in the NCI container environments + the paths are actually the same, independent of the compiler used). + This adds netcdf and pfunit definitions. :param linker: the linker instance to setup """ @@ -100,4 +99,4 @@ def _setup_linker(self, linker: Linker) -> None: [f"-L{pfunit_root}/lib", "-lfunit", "-lpfunit", f"-L{spack_view}/FARGPARSE-1.7/lib/", "-lfargparse", f"-L{spack_view}/GFTL_SHARED-1.8/lib", "-lgftl-shared-v2", - ], silent_replace=True) + ], silent_replace=True) From 7d99ddf0f0d71fd40943c49c43041e0b1bfa29dd Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 11:42:25 +1000 Subject: [PATCH 56/95] #292 Fix pfunit build. --- lfric_build/lfric_base.py | 3 --- lfric_build/pfunit_mixin.py | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 7ad22299e..edeec5720 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -227,9 +227,6 @@ def find_source_files_step( self.configurator_step() path_filter_list = list(path_filters) if path_filters else [] - # If testing is used (via LFRicBaseWithTest), unit-test will - # be handled there. - path_filter_list.append(Exclude('unit-test', '/test/')) super().find_source_files_step(path_filters=path_filter_list) self.templaterator_step(self.config) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index b4c382a37..d12f40cb1 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -156,10 +156,12 @@ def preprocess_pfunit_step(self) -> None: .with_suffix(".F90")) output_fpath.parent.mkdir(parents=True, exist_ok=True) pfunit.process(pf_file, output_fpath) + self.config.artefact_store.add(ArtefactSet.FORTRAN_COMPILER_FILES, + output_fpath) test_list = self.config.build_output / "unit-test" / "test_list.inc" with test_list.open("w", encoding="utf-8") as f: for test_name in all_tests: - f.write(f"ADD_TEST_SUITE({test_name})\n") + f.write(f"ADD_TEST_SUITE({test_name}_suite)\n") # TODO: That should be path-specific self.add_preprocessor_flags([f"-D_TEST_SUITES=\"{test_list.name}\"", From cd3db4c5fa7e0a491b326bd134f1ccfb216015b7 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 6 May 2026 12:37:51 +1000 Subject: [PATCH 57/95] #292 Fix pfunit build by disabling a warning (needs path-specific flags in order to only suppress for one file). --- lfric_build/site_specific/nci_gadi/config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 2338085a1..8ab1ec485 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -71,6 +71,17 @@ def setup_intel_llvm(self, build_config: BuildConfig) -> None: ''' super().setup_intel_llvm(build_config) tr = ToolRepository() + compiler = tr.get_tool(Category.FORTRAN_COMPILER, "ifx") + if not self.args.no_test: + # TODO: path-specific flags required here. + # pfunit driver triggers an error " #7977: The type of the + # function reference does not match the type of the function + # definition" + # for the call suite%addTest(skeleton_test_suite()) in the + # driver. This flag should only be set for + # unit-test/driver_.f90: + compiler.add_flags(["-warn", "nointerfaces"], "base") + linker = tr.get_tool(Category.LINKER, "linker-ifx") # Add netcdf and pfunit flags self._setup_linker(linker) From f676f43452c292075752dacae9d002d5225e88d9 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sat, 9 May 2026 12:21:35 +1000 Subject: [PATCH 58/95] #292 Use new public names for some Fab attributes. --- lfric_build/pfunit_mixin.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index d12f40cb1..fd1ee96ce 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -125,12 +125,11 @@ def find_source_files_step( pfunit = self.config.tool_box.get_tool(Category.PFUNIT) driver_f90 = pfunit.get_driver_f90() - # TODO: fab_base needs a `name` property driver_f90 = driver_f90.replace("program main", - f"program {self._name}_unit_test") + f"program {self.name}_unit_test") out_driver = (self.config.build_output / "unit-test" / - f"driver_{self._name}.F90") + f"driver_{self.name}.F90") out_driver.parent.mkdir(parents=True, exist_ok=True) with out_driver.open("w", encoding='utf-8') as f: f.write(driver_f90) @@ -166,8 +165,7 @@ def preprocess_pfunit_step(self) -> None: # TODO: That should be path-specific self.add_preprocessor_flags([f"-D_TEST_SUITES=\"{test_list.name}\"", "-I", str(pfunit.get_include_path())]) - # TODO: fab_base needs a `name` property - self._root_symbol.append(f"{self._name}_unit_test") + self.root_symbols.append(f"{self.name}_unit_test") compiler = self.config.tool_box.get_tool(Category.FORTRAN_COMPILER) compiler.add_flags(["-I", str(pfunit.get_include_path())]) From 8ebad2fa324d13a3349ac03bb1c69e0de03d2e07 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sat, 9 May 2026 14:37:15 +1000 Subject: [PATCH 59/95] #292 Abort build if OpenMP is disabled, since LFRic does not link without openmp. --- lfric_build/lfric_base.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index edeec5720..56b74650d 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -13,6 +13,7 @@ """ import argparse +import logging from pathlib import Path import sys from typing import List, Optional, Iterable, Union @@ -25,6 +26,10 @@ from configurator import configurator from templaterator import Templaterator +# Add a logger and connect it to stdout. +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler(sys.stdout)) + class LFRicBase(FabBase): ''' @@ -133,6 +138,22 @@ def define_command_line_options( return parser + def handle_command_line_options(self, + parser: argparse.ArgumentParser) -> None: + ''' + Make sure that openmp is not disabled, since LFRic will not build + without (because some files use openmp without openmp sentinels). + + :param argparse.ArgumentParser parser: the argument parser. + ''' + super().handle_command_line_options(parser) + + if not self.args.openmp: + logger.error("LFRic required OpenMP in order to compile and " + "link. Remove the '-no-omp` flag from the " + "command line.") + sys.exit(-1) + def setup_site_specific_location(self): ''' This method adds the required directories for site-specific From 7b5f88591545b81c45679d8ac052126921988960 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sat, 9 May 2026 16:48:11 +1000 Subject: [PATCH 60/95] #292 Fixed tests (missing app_dir, linkger without netcdf support, trying to use no-omp) --- lfric_build/tests/lfric_base_test.py | 56 +++++++++++++++------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index e5a127780..44f17a934 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -80,11 +80,12 @@ def stub_c_compiler_init() -> CCompiler: @pytest.fixture(name="stub_linker", scope='function') -def stub_linker_init(stub_c_compiler) -> Linker: +def stub_linker_init(stub_fortran_compiler) -> Linker: """ Provides a minimal linker. """ - linker = Linker(stub_c_compiler, None, 'sln') + linker = Linker(stub_fortran_compiler, None, 'sln') + linker.add_lib_flags("netcdf", ["-L", "/netcdf"]) return linker @@ -138,8 +139,11 @@ def setup_tool_repository(stub_fortran_compiler, stub_c_compiler, tr[category] = [] # Add compilers and linkers, and mark them all as available, - # as well as supporting MPI and OpenMP - for tool in [stub_c_compiler, stub_fortran_compiler, stub_linker]: + # as well as supporting MPI and OpenMP. It is important to + # add the linker first (since for each compiler a corresponding + # linker will be created, and only the stub linker specified NetCDF + # settings, without which LFRicBase will abort). + for tool in [stub_linker, stub_c_compiler, stub_fortran_compiler]: tool._mpi = True tool._openmp_flag = "-some-openmp-flag" tool._is_available = True @@ -159,18 +163,20 @@ def test_constructor(monkeypatch) -> None: Tests constructor. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test_name") + lfric_base = LFRicBase(name="test_name", apps_dir=Path(".")) # Check root symbol defaults to name if not specified assert lfric_base.root_symbol == ["test_name"] # Check root symbol can be specified lfric_base = LFRicBase(name="test_name", + apps_dir=Path("."), root_symbol="root1") assert lfric_base.root_symbol == ["root1"] # Check root symbol list lfric_base = LFRicBase(name="test_name", + apps_dir=Path("."), root_symbol=["root1", "root2"]) assert lfric_base.root_symbol == ["root1", "root2"] @@ -221,7 +227,7 @@ def create_frame_info(filename): monkeypatch.setattr('inspect.stack', lambda: mock_stack) monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) # Verify core root is set correctly assert lfric_base.lfric_core_root == mock_core @@ -239,7 +245,7 @@ def test_precision_definition_without_default(monkeypatch) -> None: "--precision_other", "32"]) monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) lfric_base.define_preprocessor_flags_step() flags = lfric_base.preprocess_flags_common @@ -260,7 +266,7 @@ def test_preprocessor_flags(monkeypatch, no_xios, mpi) -> None: Tests setting of preprocessor flags, and also that we get the expected defaults for the precision variables. """ - argv = ["fab_script", "--no-openmp"] + argv = ["fab_script"] if no_xios: argv.append("--no-xios") if not mpi: @@ -272,7 +278,7 @@ def test_preprocessor_flags(monkeypatch, no_xios, mpi) -> None: fc = tr.get_tool(Category.FORTRAN_COMPILER, "sfc") monkeypatch.setattr(fc, "_mpi", mpi) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) lfric_base.define_preprocessor_flags_step() expected_flags = [ @@ -293,7 +299,7 @@ def test_setup_site_specific_location(monkeypatch) -> None: Tests site specific path setup for LFRicBase. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) old_path = sys.path.copy() lfric_base.setup_site_specific_location() @@ -313,7 +319,7 @@ def test_get_linker_flags(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) flags = lfric_base.get_linker_flags() expected_libs = ['yaxt', 'xios', 'netcdf', 'hdf5'] @@ -333,7 +339,7 @@ def test_grab_files_step(monkeypatch) -> None: # Setup mocks monkeypatch.setattr('lfric_base.grab_folder', mock_grab) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) monkeypatch.setattr(lfric_base, '_lfric_core_root', mock_core) # Call method under test @@ -377,13 +383,11 @@ def test_find_source_files_step(monkeypatch) -> None: # Create mocks with (mock.patch('lfric_base.FabBase.find_source_files_step') as find_step, mock.patch('lfric_base.LFRicBase.templaterator_step') as temp_step, - mock.patch('lfric_base.LFRicBase.configurator_step') as conf_step, - mock.patch('lfric_base.Exclude') as mock_exclude): - lfric_base = LFRicBase(name="test") + mock.patch('lfric_base.LFRicBase.configurator_step') as conf_step): + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) lfric_base.find_source_files_step() - # Verify exclusion filter added and super called - mock_exclude.assert_called_once_with('unit-test', '/test/') + # Verify super called find_step.assert_called_once() # Verify configurator and templaterator called conf_step.assert_called_once() @@ -403,7 +407,7 @@ def test_configurator_step(monkeypatch) -> None: # Set up mocks using monkeypatch monkeypatch.setattr('lfric_base.configurator', mock_config) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) monkeypatch.setattr(lfric_base, 'get_rose_meta', mock_meta) with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): @@ -455,7 +459,7 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: monkeypatch.setattr('lfric_base.SuffixFilter', lambda *args: mock_filter) # Create LFRicBase instance - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) monkeypatch.setattr(lfric_base, '_lfric_core_root', tmp_path) # Run templaterator step @@ -510,7 +514,7 @@ def test_get_rose_meta(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) assert lfric_base.get_rose_meta() is None @@ -529,7 +533,7 @@ def test_analyse_step(monkeypatch) -> None: monkeypatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', mock_analyse) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) # Mock instance methods monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) @@ -560,7 +564,7 @@ def test_analyse_step(monkeypatch) -> None: mock_preprocess.reset_mock() mock_psyclone.reset_mock() - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) @@ -590,7 +594,7 @@ def test_preprocess_x90_step(monkeypatch) -> None: mock_preproc = mock.MagicMock() monkeypatch.setattr('lfric_base.preprocess_x90', mock_preproc) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) lfric_base.preprocess_x90_step() @@ -613,7 +617,7 @@ def test_psyclone_step(monkeypatch) -> None: # Set up monkeypatch for module level import monkeypatch.setattr('lfric_base.psyclone', mock_psy) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) # Patch instance methods. Return a copy to avoid that # PSyclone modified these lists in the lambdas when it modifies the list @@ -640,7 +644,7 @@ def test_get_psyclone_config(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) config_args = lfric_base.get_psyclone_config() assert config_args == str(lfric_base.config.source_root / @@ -654,7 +658,7 @@ def test_get_transformation_script(monkeypatch, tmp_path) -> None: monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) # Create LFRicBase instance with mocked site/platform - lfric_base = LFRicBase(name="test") + lfric_base = LFRicBase(name="test", apps_dir=Path(".")) # Create mock config config = mock.MagicMock() From 5ac67e9f6e080fca9f0ace01fc5b8b782804690e Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Sat, 9 May 2026 17:20:57 +1000 Subject: [PATCH 61/95] #292 Added test for aborting when trying to disable OpenMP with lfric. --- lfric_build/lfric_base.py | 4 ++-- lfric_build/tests/lfric_base_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 56b74650d..bd74d5d93 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -149,8 +149,8 @@ def handle_command_line_options(self, super().handle_command_line_options(parser) if not self.args.openmp: - logger.error("LFRic required OpenMP in order to compile and " - "link. Remove the '-no-omp` flag from the " + logger.error("LFRic requires OpenMP in order to compile and " + "link. Remove the '-no-omp' flag from the " "command line.") sys.exit(-1) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 44f17a934..09c869ea3 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -233,6 +233,30 @@ def create_frame_info(filename): assert lfric_base.lfric_core_root == mock_core +def test_require_openmp(monkeypatch, caplog) -> None: + ''' + Tests that using `-no-openmp` will abort with correct + error message. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py", + "--no-openmp"]) + + with pytest.raises(SystemExit): + LFRicBase(name="test", apps_dir=Path(".")) + + assert len(caplog.records) == 2 + + assert caplog.records[0].levelname == "INFO" + # Check for the details about site-specific config that is + # being imported. + assert "Imported '" in caplog.text + assert "site_specific/default/config.py" in caplog.text + + assert caplog.records[1].levelname == "ERROR" + assert ("LFRic requires OpenMP in order to compile and link. Remove " + "the '-no-omp' flag from the command line." in caplog.text) + + def test_precision_definition_without_default(monkeypatch) -> None: ''' Tests specification of precision if no default precision is From 4fc519ee52a9f4a85a52b8b01b3f0f41f3820992 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 14 May 2026 16:12:27 +1000 Subject: [PATCH 62/95] #292 Include the files from components/science/unit-test. --- lfric_build/pfunit_mixin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index fd1ee96ce..d5c871dd3 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -96,6 +96,12 @@ def grab_files_step(self) -> None: if (not self.args.no_test) and (self.apps_dir / unit_test).is_dir(): grab_folder(self.config, src=self.apps_dir / unit_test, dst_label=unit_test) + # Merge in the unit-testing files from components/science/unit-test + grab_folder(self.config, + src=(self.lfric_core_root / "components" / + "science" / "unit-test"), + dst_label=unit_test) + self._has_test = True def find_source_files_step( From a7798604bd2cda0cdbfeae6c434ba8d53da60485 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 15 May 2026 01:39:04 +1000 Subject: [PATCH 63/95] #292 Only copy .f90 files from components/science/unit-tests. --- lfric_build/pfunit_mixin.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index d5c871dd3..9d8cc8eeb 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -96,11 +96,23 @@ def grab_files_step(self) -> None: if (not self.args.no_test) and (self.apps_dir / unit_test).is_dir(): grab_folder(self.config, src=self.apps_dir / unit_test, dst_label=unit_test) - # Merge in the unit-testing files from components/science/unit-test - grab_folder(self.config, - src=(self.lfric_core_root / "components" / - "science" / "unit-test"), - dst_label=unit_test) + # Some tests also need the .f90 files from + # components/science/unit-tests, but not the .pf files. So, only + # pick the directories that contain f90 files (picking all files, + # including .pf, would add these tests to each unit-test, and + # besides being not intended, might not even compile in + # lfric_apps). + core_test_dir = (self.lfric_core_root / "components" / "science" / + "unit-test") + dirs = set() + for path in core_test_dir.rglob("*90"): + dirs.add(path.parent) + for path in dirs: + # Store the files in the corresponding subdirectories + dst = path.relative_to(core_test_dir) + grab_folder(self.config, + src=path, + dst_label=unit_test / dst) self._has_test = True From cccfeac0e815d75ae72603ed76c8e47b8a8e9ebe Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 15 May 2026 14:47:06 +1000 Subject: [PATCH 64/95] #292 Added explanations. --- lfric_build/pfunit_mixin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index 9d8cc8eeb..b4461bbe5 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -108,7 +108,9 @@ def grab_files_step(self) -> None: for path in core_test_dir.rglob("*90"): dirs.add(path.parent) for path in dirs: - # Store the files in the corresponding subdirectories + # Store the files in the corresponding subdirectories (without + # this when rsync-ing `a` and `a/b` you end up with duplicated + # files). dst = path.relative_to(core_test_dir) grab_folder(self.config, src=path, From 40961e6dcc7be6081352d1dde4190f97343ff9b0 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 21 May 2026 18:04:24 +1000 Subject: [PATCH 65/95] #292 Rename apps_dir to app_dir. --- applications/skeleton/fab_skeleton.py | 6 +++--- lfric_build/lfric_base.py | 10 +++++----- lfric_build/pfunit_mixin.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/applications/skeleton/fab_skeleton.py b/applications/skeleton/fab_skeleton.py index cd4cfc0ae..9fade7177 100755 --- a/applications/skeleton/fab_skeleton.py +++ b/applications/skeleton/fab_skeleton.py @@ -37,8 +37,8 @@ class FabSkeleton(PfUnitMixin, LFRicBase): def __init__(self, name: str = "skeleton") -> None: - apps_dir = Path(__file__).parent - super().__init__(name=name, apps_dir=apps_dir) + app_dir = Path(__file__).parent + super().__init__(name=name, app_dir=app_dir) # Store the root of this apps for later this_file = Path(__file__).resolve() self._this_root = this_file.parent @@ -48,7 +48,7 @@ def grab_files_step(self) -> None: Grabs the required source files and optimisation scripts. """ super().grab_files_step() - grab_folder(self.config, src=self.apps_dir / "source", + grab_folder(self.config, src=self.app_dir / "source", dst_label='') # Copy the optimisation scripts into a separate directory diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index bd74d5d93..51162728c 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -37,18 +37,18 @@ class LFRicBase(FabBase): :param name: the name to be used for the workspace. Note that the name of the compiler will be added to it. - :param apps_dir: the base directory of the application. + :param app_dir: the base directory of the application. :param root_symbol: the symbol (or list of symbols) of the main programs. Defaults to the parameter `name` if not specified. ''' # pylint: disable=too-many-instance-attributes def __init__(self, name: str, - apps_dir: Path, + app_dir: Path, root_symbol: Optional[Union[List[str], str]] = None ): - self._apps_dir = apps_dir + self._app_dir = app_dir # Will be set to true if a unit-test directory is found # List of all precision preprocessor symbols and their default. @@ -87,11 +87,11 @@ def __init__(self, name: str, raise RuntimeError(msg) from err @property - def apps_dir(self) -> Path: + def app_dir(self) -> Path: """ :returns: the root directory of the application. """ - return self._apps_dir + return self._app_dir @property def lfric_core_root(self) -> Path: diff --git a/lfric_build/pfunit_mixin.py b/lfric_build/pfunit_mixin.py index b4461bbe5..527903071 100755 --- a/lfric_build/pfunit_mixin.py +++ b/lfric_build/pfunit_mixin.py @@ -29,7 +29,7 @@ class PfUnitMixin: :param name: the name to be used for the workspace. Note that the name of the compiler will be added to it. - :param apps_dir: the base directory of the application. + :param app_dir: the base directory of the application. :param root_symbol: the symbol (or list of symbols) of the main programs. Defaults to the parameter `name` if not specified. @@ -40,12 +40,12 @@ class PfUnitMixin: # pylint: disable=too-many-instance-attributes def __init__(self, name: str, - apps_dir: Path, + app_dir: Path, root_symbol: Optional[Union[List[str], str]] = None ): self._has_test = False - super().__init__(name, apps_dir=apps_dir, root_symbol=root_symbol) + super().__init__(name, app_dir=app_dir, root_symbol=root_symbol) def define_command_line_options( self, @@ -93,8 +93,8 @@ def grab_files_step(self) -> None: unit_test = "unit-test" # Check if there are unit tests - if (not self.args.no_test) and (self.apps_dir / unit_test).is_dir(): - grab_folder(self.config, src=self.apps_dir / unit_test, + if (not self.args.no_test) and (self.app_dir / unit_test).is_dir(): + grab_folder(self.config, src=self.app_dir / unit_test, dst_label=unit_test) # Some tests also need the .f90 files from # components/science/unit-tests, but not the .pf files. So, only From d8573cf135be72e6b0a0c0ecd188d9ba829bb430 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 27 May 2026 12:47:35 +1000 Subject: [PATCH 66/95] #371 Added first working version of transmute support. --- lfric_build/lfric_base.py | 106 ++++++++++++---- lfric_build/psyclone_config.py | 216 +++++++++++++++++++++++++++++++++ lfric_build/psyclone_info.yaml | 42 +++++++ 3 files changed, 343 insertions(+), 21 deletions(-) create mode 100755 lfric_build/psyclone_config.py create mode 100644 lfric_build/psyclone_info.yaml diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 51162728c..26c717d0d 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -14,21 +14,22 @@ import argparse import logging +import os from pathlib import Path import sys -from typing import List, Optional, Iterable, Union +from typing import cast, Optional, Iterable, Union from fab.api import (ArtefactSet, BuildConfig, Category, Exclude, grab_folder, - Include, input_to_output_fpath, preprocess_x90, psyclone, - step, SuffixFilter) + Include, input_to_output_fpath, Linker, preprocess_x90, + psyclone, psyclone_transmute, step, SuffixFilter) from fab.fab_base.fab_base import FabBase from configurator import configurator from templaterator import Templaterator +from psyclone_config import PsycloneConfig, PsycloneInfo # Add a logger and connect it to stdout. -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) +logger = logging.getLogger("fab") class LFRicBase(FabBase): @@ -45,7 +46,7 @@ class LFRicBase(FabBase): # pylint: disable=too-many-instance-attributes def __init__(self, name: str, app_dir: Path, - root_symbol: Optional[Union[List[str], str]] = None + root_symbol: Optional[Union[list[str], str]] = None ): self._app_dir = app_dir @@ -79,6 +80,7 @@ def __init__(self, name: str, mpi=self.config.mpi, openmp=self.config.openmp, enforce_fortran_linker=True) + linker = cast(Linker, linker) try: linker.get_lib_flags("netcdf") except RuntimeError as err: @@ -86,6 +88,17 @@ def __init__(self, name: str, f"has no NetCDF library setting defined. Aborting.") raise RuntimeError(msg) from err + self._psyclone_config = PsycloneConfig(self) + if self.args.psyclone_info: + info_list = [Path(i) for i in self.args.psyclone_info] + else: + info_list = [Path(self.lfric_core_root / "lfric_build" / + "psyclone_info.yaml")] + for psy_info_file in info_list: + logger.info(f"Reading PSyclone configuration file " + f"'{psy_info_file}'.") + self._psyclone_config.read(Path(psy_info_file)) + @property def app_dir(self) -> Path: """ @@ -119,6 +132,11 @@ def define_command_line_options( '--no-xios', action="store_true", default=False, help="Disable compilation with XIOS.") + parser.add_argument( + '--psyclone-info', action="append", + help="PSyclone configuration files, controlling when to " + "run PSyclone.") + # Precision related command line arguments # ---------------------------------------- group = parser.add_argument_group( @@ -178,7 +196,7 @@ def define_preprocessor_flags_step(self) -> None: - Use of XIOS (if not disabled using --no-xios command line option) - Disabling MPI (if disabled using --no-mpi) ''' - preprocessor_flags: List[str] = [] + preprocessor_flags: list[str] = [] # Check all required precision defines for prec_name, _ in self._all_precisions: @@ -198,7 +216,7 @@ def define_preprocessor_flags_step(self) -> None: self.add_preprocessor_flags(preprocessor_flags) - def get_linker_flags(self) -> List[str]: + def get_linker_flags(self) -> list[str]: ''' This method overwrites the base class get_linker_flags. It passes the libraries that LFRic uses to the linker. Currently, these libraries @@ -356,7 +374,8 @@ def preprocess_x90_step(self) -> None: def psyclone_step( self, ignore_dependencies: Optional[Iterable[str]] = None, - additional_parameters: Optional[list[str]] = None + additional_parameters: Optional[list[str]] = None, + kernel_roots: Optional[list[Path]] = None, ) -> None: ''' This method runs Fab's psyclone. It first sets the psyclone @@ -367,24 +386,67 @@ def psyclone_step( got through calling `get_transformation_script`, the api, and the additional psyclone command line arguments. - :param ignore_dependencies: + :param ignore_dependencies: Third party Fortran module names in + USE statements, 'DEPENDS ON' files and modules to be ignored. :param additional_parameters: optional additional parameter for the PSyclone. + :param kernel_roots: + Folders containing kernel files. Must be part of the analysed + source code. ''' + kernel_roots = kernel_roots or [] psyclone_cli_args = ["--config", self.get_psyclone_config()] if additional_parameters: psyclone_cli_args.extend(additional_parameters) - # To avoid impacting other code, store the original search path - old_sys_path = sys.path[:] - sys.path.extend(self._add_python_paths) - psyclone(self.config, kernel_roots=[(self.config.build_output / - "kernel")], - transformation_script=self.get_transformation_script, - api="lfric", - cli_args=psyclone_cli_args, - ignore_dependencies=ignore_dependencies) - sys.path = old_sys_path + add_python_paths = ":".join(str(i) for i in self._add_python_paths) + for phase in self._psyclone_config.all_phases: + logger.info(f"Running PSyclone phase {phase}.") + psyclone_info = self._psyclone_config.get_info(phase) + # To avoid impacting other code, store the original search path + # We have to modify PYTHONPATH (and not sys.path), since PSycline + # is run in its own shell (i.e. it inherits PYTHONPATH, but not + # sys.path). + orig_pythonpath = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = (f"{psyclone_info.opt_path}:" + f"{add_python_paths}:" + f"{orig_pythonpath}") + if psyclone_info.api: + psyclone(self.config, + kernel_roots=(kernel_roots + + [self.config.build_output / "kernel"]), + transformation_script=psyclone_info.get_script, + api=psyclone_info.api, + cli_args=psyclone_cli_args, + ignore_dependencies=ignore_dependencies) + else: + self._psyclone_transmute(psyclone_info, + psyclone_cli_args) + # Reset PYTHONPATH + os.environ["PYTHONPATH"] = orig_pythonpath + + def _psyclone_transmute(self, + psyclone_info: PsycloneInfo, + psyclone_cli_args: list[str], + ): + f90_files: list[Path] = [] + af_store = self.config.artefact_store + for file in af_store[ArtefactSet.FORTRAN_COMPILER_FILES]: + script = psyclone_info.get_script(file, self.config) + if script: + f90_files.append(file) + + # Don't use a suffix, meaning the original source files will be + # overwritten. Otherwise, PSyclone might find two files with the + # same kernels (once transmuted, one original) and abort. + psyclone_transmute( + self.config, + fortran_files=f90_files, + transformation_script=psyclone_info.get_script, + cli_args=psyclone_cli_args, + suffix="", + artefact_set=ArtefactSet.FORTRAN_COMPILER_FILES + ) def get_psyclone_config(self) -> str: ''' @@ -411,10 +473,12 @@ def get_transformation_script(self, fpath: Path, :returns: the transformation script to be used by PSyclone. ''' # Newer LFRic versions have a psykal directory + logger.info(f"getting script '{fpath}' config: " + f"'{str(self._psyclone_config)}'") optimisation_path = (config.source_root / "optimisation" / f"{self.site}-{self.platform}" / "psykal") relative_path = None - # The soure file might be either in build_output (e.g. a preprocessed + # The source file might be either in build_output (e.g. a preprocessed # .X90 file), or still in source (.x90 file). Check if the file # is in one of the two sub-trees, and use the relative path to # check if there is a file-specific optimisation script diff --git a/lfric_build/psyclone_config.py b/lfric_build/psyclone_config.py new file mode 100755 index 000000000..fc33ea757 --- /dev/null +++ b/lfric_build/psyclone_config.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +''' +This module reads in a psyclone_info.yaml file. +''' +from pathlib import Path +from typing import Optional, Union +import yaml + +from fab.api import BuildConfig +from fab.fab_base.fab_base import FabBase + + +class PsycloneInfo: + + FILE_SPECIFIC = "file_specific" + EXCLUDE = "exclude" + + def __init__(self, name: str, fab_base: FabBase) -> None: + self._fab_base = fab_base + # This will be initialised/updated each time when reading an info file. + self._opt_path = Path() + self._script_dir: str = "" + self._name: str = name + self._comment: str = "" + self._api: str = "" + self._artefacts: str = "" + self._rules: list[tuple[str, list[str]]] = [] + + def __str__(self) -> str: + return self._name + + @property + def name(self) -> str: + """ + :returns: the name of this phas. + """ + return self._name + + @property + def comment(self) -> str: + """ + :returns: the comment for this phase. + """ + return self._comment + + @property + def api(self) -> str: + """ + :returns: the PSyclone command line options. + """ + return self._api + + @property + def artefacts(self) -> str: + """ + :returns: the artefacts to apply this info to. + """ + return self._artefacts + + @property + def opt_path(self) -> Path: + return self._opt_path + + def update(self, info: dict[str, str]) -> None: + """ + Update this PSyclone information with data taken from a + PSyclone info yaml file. This function is used to initially + read in a first specification, or update a specification based + on an additional file being read in later. + + :param info: the yaml information taken from a PSyclone info file. + """ + for rule in info: + if rule == "comment": + self._comment = info["comment"] + elif rule == "api": + self._api = info["api"] + elif rule == "artefacts": + self._artefacts = info["artefacts"] + elif rule == "script_dir": + self._script_dir = info["script_dir"] + else: + self._read_rule(rule, info[rule]) + + # Store the potentially updated optimisation root path, i.e. the site- + # and platform-specific location, followed by a script dir (typically + # transmute or psykal). This path is used in a few places. + self._opt_path = (self._fab_base.config.source_root / "optimisation" / + f"{self._fab_base.site}-{self._fab_base.platform}" / + self._script_dir) + + def _read_rule(self, rule: str, file_list: str) -> None: + """ + """ + # Support '*', which is a reserved character in yaml and needs to + # be escaped or quoted. + if file_list in ["\\*", "'*'", '"*"']: + file_list = "*" + self._rules.append((rule, file_list.split())) + + def view(self) -> str: + s = f"""{self._name}: +comment: {self.comment} +api: {self.api} +artefacts: {self.artefacts} +script_dir: {self._script_dir} +rules: {self._rules} +""" + return s + + def file_specific_script(self, fpath: Path) -> Optional[Path]: + relative_path = None + # The source file might be either in build_output (e.g. a preprocessed + # .X90 file), or still in source (.x90 file). Check if the file + # is in one of the two sub-trees, and use the relative path to + # check if there is a file-specific optimisation script + for base_path in [self._fab_base.config.source_root, + self._fab_base.config.build_output]: + try: + relative_path = fpath.relative_to(base_path) + except ValueError: + # The file is not under the `base_path` - keep on checking + pass + + if relative_path: + # The file was under either source or build. Check if there + # is a file-specific optimisation script: + local_transformation_script = (self.opt_path / + (relative_path.with_suffix('.py'))) + if local_transformation_script.exists(): + return local_transformation_script + return None + + def get_script(self, file: Path, config: BuildConfig) -> Optional[Path]: + # Search starting from the end, so last rule wins + file_str = str(file) + + for rule, file_list in self._rules[::-1]: + for pattern in file_list: + if pattern not in file_str and pattern != "*": + continue + + # Now the pattern matches. Check which rule is used + # (note that file_specific might fall through in case that + # there is no file-specific script) + if rule == PsycloneInfo.FILE_SPECIFIC: + script = self.file_specific_script(file) + if script: + return script + if pattern == "*": + # Fall through, i.e. check for other rules + continue + + # Now we have an explicit request for a file-specific + # script a file, but that script does not exist. + raise FileNotFoundError( + f"Cannot find explicitly requested script '{script}'.") + + elif rule == PsycloneInfo.EXCLUDE: + # Exclude pattern matches: + return None + + else: + opt_script = self.opt_path / rule + if not opt_script.exists(): + raise FileExistsError(f"Cannot find script " + f"'{opt_script}'.") + return opt_script + + return None + + +class PsycloneConfig: + + def __init__(self, fab_base: FabBase) -> None: + self._fab_base = fab_base + self._all_phases: list[str] = [] + self._psyclone_info: dict[str, PsycloneInfo] = {} + + @property + def all_phases(self): + return self._all_phases + + def get_info(self, phase: str) -> PsycloneInfo: + return self._psyclone_info[phase] + + def view(self): + s = f"""Phases: {" ".join(self._all_phases)}\n\n""" + for phase in self._all_phases: + s += f"{self._psyclone_info[phase].view()}\n" + return s + + def read(self, filename: Union[str, Path]) -> None: + + with open(filename, "r", encoding="utf8") as stream: + dependencies = yaml.safe_load(stream) + + # First take phases (if available) + if dependencies.get("phases", None): + self._all_phases = dependencies["phases"] + + for key in dependencies: + if key == "phases": + # Already handled + continue + if key not in self._psyclone_info: + self._psyclone_info[key] = PsycloneInfo(key, self._fab_base) + + self._psyclone_info[key].update(dependencies[key]) diff --git a/lfric_build/psyclone_info.yaml b/lfric_build/psyclone_info.yaml new file mode 100644 index 000000000..98e0702d6 --- /dev/null +++ b/lfric_build/psyclone_info.yaml @@ -0,0 +1,42 @@ +# This file controls which files are being processed by PSyclone, +# both using PSyclone's transmute ability (transformation generic +# Fortran files), and its DSL (kernel based) capability. + +# First we specify the phases for PSyclone. This allows +# any apps to run transmute and DSL steps in any order just by +# changing (or supplying a different) psyclone-info file. +# In this example, we run a transmute step before and after the DSL step. +# This allows transforming source code before PSyclone runs its DSL +# processing (during which transformed source code might be inlined). +# The third step is done to support feedback from the DSL processing +# to trigger additional processing (for example, if the DSL processing +# adds OpenACC directive, it might determine that additional source +# files need to be marked up to be compiled for OpenACC, which can then +# be done in an additional transmute phase). + +phases: + # Just a single phase: running PSyclone in DSL mode + - dsl + +dsl: + comment: "PSylone DSL Phase" + api: lfric + + # Run on all x90 files + artefacts: x90 + + script_dir: psykal + + # The first two directives reproduce the default PSyclone triggering used + # in LFRic: + + # Run optimisation/.../global.py on all x90 files (x90 because this is + # the dsl section, as specified in the artefacts above) + global.py: \* + + # This represents existing functionality: if there is a file-specific .py + # file, use it. The specification of '*' does not trigger an error + # if there is no file_specific script (while explicitly putting a name + # here as shown in the previous pre_dsl phase will trigger an error, to + # catch typos early) + file_specific: \* From da94e4400de91b644d585ac14a7d7d681a22f180 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 27 May 2026 12:53:54 +1000 Subject: [PATCH 67/95] #292 Removed outdated comment. --- lfric_build/lfric_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 51162728c..eb666b46c 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -49,7 +49,6 @@ def __init__(self, name: str, ): self._app_dir = app_dir - # Will be set to true if a unit-test directory is found # List of all precision preprocessor symbols and their default. # Used to add corresponding command line options, and then to define From e3b7dd74ab37a31b97bf21966f5aa87ced19c40f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 00:54:52 +1000 Subject: [PATCH 68/95] #371 Renmaed psyclone_config to psyclone_control. --- lfric_build/lfric_base.py | 20 ++-- ...psyclone_config.py => psyclone_control.py} | 104 ++++++++++++++++-- 2 files changed, 106 insertions(+), 18 deletions(-) rename lfric_build/{psyclone_config.py => psyclone_control.py} (64%) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 672dca672..c197bbae2 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -26,7 +26,7 @@ from configurator import configurator from templaterator import Templaterator -from psyclone_config import PsycloneConfig, PsycloneInfo +from psyclone_control import PsycloneControl, PsycloneInfo # Add a logger and connect it to stdout. logger = logging.getLogger("fab") @@ -87,16 +87,18 @@ def __init__(self, name: str, f"has no NetCDF library setting defined. Aborting.") raise RuntimeError(msg) from err - self._psyclone_config = PsycloneConfig(self) + self._psyclone_control = PsycloneControl(self) if self.args.psyclone_info: info_list = [Path(i) for i in self.args.psyclone_info] else: + # This default rule implements the "file-specific if exists, + # otherwise global.py" rule. info_list = [Path(self.lfric_core_root / "lfric_build" / "psyclone_info.yaml")] for psy_info_file in info_list: logger.info(f"Reading PSyclone configuration file " f"'{psy_info_file}'.") - self._psyclone_config.read(Path(psy_info_file)) + self._psyclone_control.read(Path(psy_info_file)) @property def app_dir(self) -> Path: @@ -399,14 +401,18 @@ def psyclone_step( psyclone_cli_args.extend(additional_parameters) add_python_paths = ":".join(str(i) for i in self._add_python_paths) - for phase in self._psyclone_config.all_phases: + for phase in self._psyclone_control.all_phases: logger.info(f"Running PSyclone phase {phase}.") - psyclone_info = self._psyclone_config.get_info(phase) + psyclone_info = self._psyclone_control.get_info(phase) # To avoid impacting other code, store the original search path - # We have to modify PYTHONPATH (and not sys.path), since PSycline + # We have to modify PYTHONPATH (and not sys.path), since PSyclone # is run in its own shell (i.e. it inherits PYTHONPATH, but not # sys.path). orig_pythonpath = os.environ.get("PYTHONPATH", "") + # Add various paths: optimisation/site-platform/transmute + # (=opt_path) is required for some transmute scripts that + # import helper functions. Adding add_python_paths is + # required for other, shared PSyclone scripts os.environ["PYTHONPATH"] = (f"{psyclone_info.opt_path}:" f"{add_python_paths}:" f"{orig_pythonpath}") @@ -473,7 +479,7 @@ def get_transformation_script(self, fpath: Path, ''' # Newer LFRic versions have a psykal directory logger.info(f"getting script '{fpath}' config: " - f"'{str(self._psyclone_config)}'") + f"'{str(self._psyclone_control)}'") optimisation_path = (config.source_root / "optimisation" / f"{self.site}-{self.platform}" / "psykal") relative_path = None diff --git a/lfric_build/psyclone_config.py b/lfric_build/psyclone_control.py similarity index 64% rename from lfric_build/psyclone_config.py rename to lfric_build/psyclone_control.py index fc33ea757..2507b4085 100755 --- a/lfric_build/psyclone_config.py +++ b/lfric_build/psyclone_control.py @@ -18,6 +18,15 @@ class PsycloneInfo: + """ + This class stores the set of rules and settings for one specific PSyclone + phase. + + :param name: the name of this phase. + :param fab_base: the application script derived from FabBase. Required to + get access to the build config for paths, and the selected site and + platform. + """ FILE_SPECIFIC = "file_specific" EXCLUDE = "exclude" @@ -33,9 +42,6 @@ def __init__(self, name: str, fab_base: FabBase) -> None: self._artefacts: str = "" self._rules: list[tuple[str, list[str]]] = [] - def __str__(self) -> str: - return self._name - @property def name(self) -> str: """ @@ -66,6 +72,11 @@ def artefacts(self) -> str: @property def opt_path(self) -> Path: + """ + :return: the optimisation root as absolute path (including site- + and platform-specific settings, and subdirectory, e.g. psykal + or transmute). + """ return self._opt_path def update(self, info: dict[str, str]) -> None: @@ -98,6 +109,12 @@ def update(self, info: dict[str, str]) -> None: def _read_rule(self, rule: str, file_list: str) -> None: """ + Parses a single rule from the yaml file. It especially handles + various way a '*' can be specified in a yaml file. + + :param rule: the name of the rule (typically the script name, or + special term like `exclude` or `file_specific`). + :param file_list: the list of files to which to apply the rule to. """ # Support '*', which is a reserved character in yaml and needs to # be escaped or quoted. @@ -106,6 +123,9 @@ def _read_rule(self, rule: str, file_list: str) -> None: self._rules.append((rule, file_list.split())) def view(self) -> str: + """ + :returns: a string representation of this phase in yaml format. + """ s = f"""{self._name}: comment: {self.comment} api: {self.api} @@ -116,6 +136,15 @@ def view(self) -> str: return s def file_specific_script(self, fpath: Path) -> Optional[Path]: + """ + Searches for a file-specific optimisation script. It will search + both under the source and the build directories of the project + directory. + + :param fpath: the file path of the Fortran file. + :returns: the path of the file-specific optimisation script, or + None if no such file exists. + """ relative_path = None # The source file might be either in build_output (e.g. a preprocessed # .X90 file), or still in source (.x90 file). Check if the file @@ -138,10 +167,24 @@ def file_specific_script(self, fpath: Path) -> Optional[Path]: return local_transformation_script return None - def get_script(self, file: Path, config: BuildConfig) -> Optional[Path]: - # Search starting from the end, so last rule wins - file_str = str(file) + def get_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: + """ + This method returns the script to be used for a given filename, or + None if no rule applies (or an explicit exclude rule applies) + + This function will also provided to Fab's PSyclone step, and as such + it will receive the config object, even though it is not used. + + :param fpath: the Fortran source file for which to find a + transformation script. + :param config: the build configuration (unused in this implementation) + + :returns: the path to the transformation script, or None if no rule + applies (or an exclude rule applies). + """ + file_str = str(fpath) + # Search starting from the end, so last rule wins for rule, file_list in self._rules[::-1]: for pattern in file_list: if pattern not in file_str and pattern != "*": @@ -151,7 +194,7 @@ def get_script(self, file: Path, config: BuildConfig) -> Optional[Path]: # (note that file_specific might fall through in case that # there is no file-specific script) if rule == PsycloneInfo.FILE_SPECIFIC: - script = self.file_specific_script(file) + script = self.file_specific_script(fpath) if script: return script if pattern == "*": @@ -159,7 +202,7 @@ def get_script(self, file: Path, config: BuildConfig) -> Optional[Path]: continue # Now we have an explicit request for a file-specific - # script a file, but that script does not exist. + # script, but that script does not exist. raise FileNotFoundError( f"Cannot find explicitly requested script '{script}'.") @@ -177,7 +220,18 @@ def get_script(self, file: Path, config: BuildConfig) -> Optional[Path]: return None -class PsycloneConfig: +class PsycloneControl: + """ + This class stores the information from psyclone_info.yaml file(s). Several + files can be read, and latter information will extend the rules from + previous files, and replace the phases executed. + + Details of each phase will be stored in PsycloneInfo instances. + + :param fab_base: The FabBase derived application script. This is required + to get site, platform and config information when searching for + PSyclone scripts to be executed. + """ def __init__(self, fab_base: FabBase) -> None: self._fab_base = fab_base @@ -185,19 +239,47 @@ def __init__(self, fab_base: FabBase) -> None: self._psyclone_info: dict[str, PsycloneInfo] = {} @property - def all_phases(self): + def all_phases(self) -> list[str]: + """ + :returns: the list of all PSyclone phases to execute. + """ return self._all_phases def get_info(self, phase: str) -> PsycloneInfo: + """ + Returns the PSyclone information for the specified phase. + + :param phase: the name of the phase. + + :returns: the PSyclone Information for the specified phase. + """ return self._psyclone_info[phase] - def view(self): + def view(self) -> str: + """ + This returns a string representation of the combined read yaml files. + This is useful to be logged to show the actual details used. + + :returns: the string represenation in yaml format of this PSyclone + control instance. + """ s = f"""Phases: {" ".join(self._all_phases)}\n\n""" for phase in self._all_phases: s += f"{self._psyclone_info[phase].view()}\n" return s def read(self, filename: Union[str, Path]) -> None: + """ + Reads a yaml file, and extends the potentially existing information. + Any phases specified in the new read yaml file will replace the + phases to be executed (i.e. will overwrite what was previously + specified). Any new rule sets will be added as new PsycloneInfo + instance. Rules for an exiting phase will be appended to the + existing information. The precedence handling means that any later + rule will overwrite any previous rule. + + :param filename: the filename to read. + """ with open(filename, "r", encoding="utf8") as stream: dependencies = yaml.safe_load(stream) From 9742b1f142db6e1dcdc7f5af7c048dd7bc8b9aad Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 11:36:26 +1000 Subject: [PATCH 69/95] #292 Added netcdf config object to lfric_core, as requested in the Fab review. --- lfric_build/nf_config.py | 37 ++++++++ .../default/setup_script_cray.py | 19 ++--- .../site_specific/default/setup_script_gnu.py | 19 ++--- .../default/setup_script_intel_classic.py | 20 ++--- .../default/setup_script_intel_llvm.py | 21 ++--- .../default/setup_script_nvidia.py | 20 ++--- lfric_build/site_specific/nci_gadi/config.py | 12 +-- lfric_build/tests/nf_config_test.py | 85 +++++++++++++++++++ 8 files changed, 162 insertions(+), 71 deletions(-) create mode 100644 lfric_build/nf_config.py create mode 100644 lfric_build/tests/nf_config_test.py diff --git a/lfric_build/nf_config.py b/lfric_build/nf_config.py new file mode 100644 index 000000000..4d09c30ed --- /dev/null +++ b/lfric_build/nf_config.py @@ -0,0 +1,37 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +"""This file contains the class to interface with NetCDF's nf-config script. +""" + +from typing import List + +from fab.tools.category import Category +from fab.tools.tool import Tool + + +class NfConfig(Tool): + '''This class interfaces with NetCDF's nf-config tool. It is not added + to the ToolRepository, it is intended for site-specific configurations + to make it easier to query for NetCDF settings. + ''' + + def __init__(self): + super().__init__("nf-config", "nf-config", Category.MISC) + + def get_compiler_flags(self) -> List[str]: + """ + :returns: the compilation flags to use for NetCDF. + """ + flags = self.run(additional_parameters=["--fflags"]) + return flags.split() + + def get_linker_flags(self) -> List[str]: + """ + :returns: the linker flags to use for NetCDF. + """ + flags = self.run(additional_parameters=["--flibs"]) + return flags.split() diff --git a/lfric_build/site_specific/default/setup_script_cray.py b/lfric_build/site_specific/default/setup_script_cray.py index 83c17771e..4e3d45f27 100644 --- a/lfric_build/site_specific/default/setup_script_cray.py +++ b/lfric_build/site_specific/default/setup_script_cray.py @@ -18,6 +18,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository +from nf_config import NfConfig + def setup_script_cray(build_config: BuildConfig, args: argparse.Namespace) -> None: @@ -97,20 +99,11 @@ def setup_script_cray(build_config: BuildConfig, linker = tr.get_tool(Category.LINKER, f"linker-{ftn.name}") linker = cast(Linker, linker) - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): - shell = tr.get_default(Category.SHELL) - - try: - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - except RuntimeError: - nc_flibs = [] + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags()) - linker.add_lib_flags("netcdf", nc_flibs) linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index ff4c91d08..da935bc8b 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -18,6 +18,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository +from nf_config import NfConfig + def setup_script_gnu(build_config: BuildConfig, args: argparse.Namespace) -> None: @@ -92,20 +94,11 @@ def setup_script_gnu(build_config: BuildConfig, linker = tr.get_tool(Category.LINKER, f"linker-{gfortran.name}") linker = cast(Linker, linker) - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): - shell = tr.get_default(Category.SHELL) - - try: - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - except RuntimeError: - nc_flibs = [] + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags()) - linker.add_lib_flags("netcdf", nc_flibs) linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) diff --git a/lfric_build/site_specific/default/setup_script_intel_classic.py b/lfric_build/site_specific/default/setup_script_intel_classic.py index 6995638d9..2d91411b3 100644 --- a/lfric_build/site_specific/default/setup_script_intel_classic.py +++ b/lfric_build/site_specific/default/setup_script_intel_classic.py @@ -18,6 +18,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository +from nf_config import NfConfig + def setup_script_intel_classic(build_config: BuildConfig, args: argparse.Namespace) -> None: @@ -92,19 +94,11 @@ def setup_script_intel_classic(build_config: BuildConfig, linker = tr.get_tool(Category.LINKER, f"linker-{ifort.name}") linker = cast(Linker, linker) - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): - shell = tr.get_default(Category.SHELL) - try: - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - except RuntimeError: - nc_flibs = [] - - linker.add_lib_flags("netcdf", nc_flibs) + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags()) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) diff --git a/lfric_build/site_specific/default/setup_script_intel_llvm.py b/lfric_build/site_specific/default/setup_script_intel_llvm.py index bc430283b..84167b035 100644 --- a/lfric_build/site_specific/default/setup_script_intel_llvm.py +++ b/lfric_build/site_specific/default/setup_script_intel_llvm.py @@ -18,6 +18,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository +from nf_config import NfConfig + def setup_script_intel_llvm(build_config: BuildConfig, args: argparse.Namespace) -> None: @@ -74,19 +76,12 @@ def setup_script_intel_llvm(build_config: BuildConfig, # linker-mpif90-ifx will use these flags as well. linker = tr.get_tool(Category.LINKER, f"linker-{ifx.name}") linker = cast(Linker, linker) # Make mypy happy - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): - shell = tr.get_default(Category.SHELL) - try: - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - except RuntimeError: - nc_flibs = [] - - linker.add_lib_flags("netcdf", nc_flibs) + + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags()) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) diff --git a/lfric_build/site_specific/default/setup_script_nvidia.py b/lfric_build/site_specific/default/setup_script_nvidia.py index 434b84078..b76b2af8b 100644 --- a/lfric_build/site_specific/default/setup_script_nvidia.py +++ b/lfric_build/site_specific/default/setup_script_nvidia.py @@ -18,6 +18,8 @@ from fab.api import BuildConfig, Category, Compiler, Linker, ToolRepository +from nf_config import NfConfig + def setup_script_nvidia(build_config: BuildConfig, args: argparse.Namespace) -> None: @@ -93,19 +95,11 @@ def setup_script_nvidia(build_config: BuildConfig, linker = tr.get_tool(Category.LINKER, f"linker-{nvfortran.name}") linker = cast(Linker, linker) - # ATM we don't use a shell when running a tool, and as such - # we can't directly use "$()" as parameter. So query these values using - # Fab's shell tool (doesn't really matter which shell we get, so just - # ask for the default): - shell = tr.get_default(Category.SHELL) - try: - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - except RuntimeError: - nc_flibs = [] - - linker.add_lib_flags("netcdf", nc_flibs) + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags()) + linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"]) linker.add_lib_flags("xios", ["-lxios"]) linker.add_lib_flags("hdf5", ["-lhdf5"]) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 8ab1ec485..282a322e7 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -94,13 +94,13 @@ def _setup_linker(self, linker: Linker) -> None: :param linker: the linker instance to setup """ - tr = ToolRepository() - shell = tr.get_default(Category.SHELL) - # We must remove the trailing new line, and create a list: - nc_flibs = shell.run(additional_parameters=["-c", "nf-config --flibs"], - capture_output=True).strip().split() - linker.add_lib_flags("netcdf", nc_flibs, silent_replace=True) + nf_config = NfConfig() + if nf_config.is_available: + # If not available, the site-specific setup must define netcdf + linker.add_lib_flags("netcdf", nf_config.get_linker_flags(), + silent_replace=True) + tr = ToolRepository() pfunit = tr.get_tool(Category.PFUNIT, "funitproc") pfunit_root = pfunit.get_root_path() spack_view = os.environ.get("SPACK_ENV_VIEW", "") diff --git a/lfric_build/tests/nf_config_test.py b/lfric_build/tests/nf_config_test.py new file mode 100644 index 000000000..464796f8a --- /dev/null +++ b/lfric_build/tests/nf_config_test.py @@ -0,0 +1,85 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## +""" +Tests the nf-config wrapper. +""" + +from pytest_subprocess.fake_process import FakeProcess + +from fab.tools.category import Category +from nf_config import NfConfig + + +def call_list(fake_process: FakeProcess) -> list[list[str]]: + """ + Converts FakeProcess calls to strings. + + :returns: List of argument strings per call. + """ + result: list[list[str]] = [] + for call in fake_process.calls: + result.append([str(arg) for arg in call]) + return result + + +def test_constructor() -> None: + """ + Tests default constructor. + """ + nfc = NfConfig() + assert nfc.category == Category.MISC + assert nfc.name == "nf-config" + assert nfc.exec_name == "nf-config" + assert nfc.get_flags() == [] + + +def test_nf_config_check_available(fake_process: FakeProcess) -> None: + """ + Tests availability functionality. + """ + fake_process.register(['nf-config', '--version'], + returncode=0, + stdout="netCDF-Fortran 4.6.1") + + nfc = NfConfig() + assert nfc.check_available() + assert call_list(fake_process) == [["nf-config", "--version"]] + + +def test_nf_config_check_unavailable(fake_process: FakeProcess) -> None: + """ + Tests availability failure. + """ + fake_process.register(['nf-config', '--version'], + returncode=127, + stderr="command 'nf-config' not found") + nfc = NfConfig() + assert not nfc.check_available() + assert call_list(fake_process) == [["nf-config", "--version"]] + + +def test_nf_config_compiler_flags(fake_process: FakeProcess) -> None: + """ + Tests getting the compiler flags. + """ + fake_process.register(['nf-config', '--fflags'], + returncode=0, + stdout="-I /somewhere") + nfc = NfConfig() + assert nfc.get_compiler_flags() == ["-I", "/somewhere"] + assert call_list(fake_process) == [["nf-config", "--fflags"]] + + +def test_nf_config_linker_flags(fake_process: FakeProcess) -> None: + """ + Tests availability failure. + """ + fake_process.register(['nf-config', '--flibs'], + returncode=0, + stdout="-L /somewhere -lsomewhat") + nfc = NfConfig() + assert nfc.get_linker_flags() == ["-L", "/somewhere", "-lsomewhat"] + assert call_list(fake_process) == [["nf-config", "--flibs"]] From a1a64eb8dd7edb212a0d7853b1c20662cb299806 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 11:37:13 +1000 Subject: [PATCH 70/95] #292 Removed pkg_config object, since it will be added to Fab. --- lfric_build/pkg_config.py | 134 ---------- lfric_build/tests/test_pkg_config.py | 364 --------------------------- 2 files changed, 498 deletions(-) delete mode 100644 lfric_build/pkg_config.py delete mode 100644 lfric_build/tests/test_pkg_config.py diff --git a/lfric_build/pkg_config.py b/lfric_build/pkg_config.py deleted file mode 100644 index 7eb4f500c..000000000 --- a/lfric_build/pkg_config.py +++ /dev/null @@ -1,134 +0,0 @@ -############################################################################## -# (c) Crown copyright 2025 Met Office. All rights reserved. -# The file LICENCE, distributed with this code, contains details of the terms -# under which the code may be used. -############################################################################## -""" -Makes use of pkg-config files to find out about libraries. -""" -from os import environ as os_environ -from enum import StrEnum -from re import match as re_match -from subprocess import run, PIPE -from typing import Dict, Iterable, List, Tuple, Union - - -class LinkType(StrEnum): - SHARED = '--shared' - STATIC = '--static' - - -class PackageException(Exception): - pass - - -class Package: - """ - Holds details of a library. - """ - def __init__(self, specification: str, - link_type: LinkType = LinkType.SHARED): - """ - Constructs Package object from details held in pkg-config files. - - Version requirement may be specified in the same way pkg-config - understands them. i.e. =, >, <, >= and <= - - :param specification: Package name with optional version requirement. - :raises PackageException: A package fulfilling the specification was - not found. - """ - match = re_match(r'(\w+)[ \t<>=]?', specification) - if match: - self.__name = match.group(1) - else: - raise PackageException( - "Unable to parse specification: " + specification - ) - - self.__pkg_config(specification, ['--print-errors', '--exists']) - - version = self.__pkg_config(specification, ['--modversion']) - if version: - self.__version = tuple([int(component) - if component.isdigit() else component - for component in version[0].split('.')]) - else: # No version string - self.__version = tuple() - - compile_arguments = self.__pkg_config(specification, - ['--cflags', link_type]) - self.__compile_arguments = self.__split_arguments(compile_arguments) - - # ToDo: It may be necessary to use --pure with static link type. - # - link_arguments = self.__pkg_config(specification, - ['--libs', link_type]) - self.__link_arguments = self.__split_arguments(link_arguments) - - @staticmethod - def __pkg_config(specification: str, arguments: List[str]) -> List[str]: - environment: Dict[str, str] = {'PKG_CONFIG_DEBUG_SPEW': 'YES'} - if 'PKG_CONFIG_LIBDIR' in os_environ: - environment['PKG_CONFIG_LIBDIR'] = os_environ['PKG_CONFIG_LIBDIR'] - if 'PKG_CONFIG_PATH' in os_environ: - environment['PKG_CONFIG_PATH'] = os_environ['PKG_CONFIG_PATH'] - command = ['pkg-config'] - command.extend(arguments) - command.append(specification) - process = run(command, stdout=PIPE, stderr=PIPE, encoding='utf8', - env=environment) - if process.returncode != 0: - raise PackageException( - f"Failed to run [{command}]: {process.stderr}" - ) - if process.stderr: - raise PackageException(process.stderr) - return process.stdout.split() - - @staticmethod - def __split_arguments(arguments: Iterable[str]) -> Tuple[str, ...]: - result: List[str] = [] - directive = '' - for argument in arguments: - if argument in ['-I', '-L', '-l']: - directive = argument - continue - if directive: - result.append(directive + argument) - directive = '' - continue - result.append(argument) - return tuple(result) - - @property - def name(self) -> str: - """ - Gets library name. - """ - return self.__name - - @property - def version(self) -> Tuple[Union[int, str], ...]: - """ - Gets library version. - """ - return self.__version - - @property - def compile_arguments(self) -> Tuple[str, ...]: - """ - Gets arguments for compiler. - - Arguments are canonicalised into "no space" form. - """ - return self.__compile_arguments - - @property - def link_arguments(self) -> Tuple[str, ...]: - """ - Gets arguments for linker. - - Arguments are canonicalised into "no space" form. - """ - return self.__link_arguments diff --git a/lfric_build/tests/test_pkg_config.py b/lfric_build/tests/test_pkg_config.py deleted file mode 100644 index b7fb0cfac..000000000 --- a/lfric_build/tests/test_pkg_config.py +++ /dev/null @@ -1,364 +0,0 @@ -############################################################################## -# (c) Crown copyright 2025 Met Office. All rights reserved. -# The file LICENCE, distributed with this code, contains details of the terms -# under which the code may be used. -############################################################################## -from pathlib import Path -from textwrap import dedent -from typing import Any, Dict, Optional, Tuple - -from pytest import MonkeyPatch, fixture, mark, raises - -from ..pkg_config import LinkType, Package, PackageException - - -@fixture -def system_path(tmp_path: Path) -> Path: - system_path = tmp_path / 'usr' / 'local' - system_path.mkdir(parents=True) - return system_path - - -@fixture -def system_pkg_path(system_path: Path, monkeypatch: MonkeyPatch) -> Path: - pkg_path = system_path / 'lib' / 'pkg_config' - pkg_path.mkdir(parents=True) - monkeypatch.setenv('PKG_CONFIG_LIBDIR', str(pkg_path)) - return pkg_path - - -@fixture -def user_path(tmp_path: Path) -> Path: - user_path = tmp_path / 'opt' / 'special' - user_path.mkdir(parents=True) - return user_path - - -@fixture -def user_pkg_path(user_path: Path, monkeypatch: MonkeyPatch) -> Path: - pkg_path = user_path / 'lib' / 'pkg_config' - pkg_path.mkdir(parents=True) - monkeypatch.setenv('PKG_CONFIG_PATH', str(pkg_path)) - return pkg_path - - -class TestPackage: - @mark.parametrize('name, expected', [ - ("single", { - 'version': (1, 0, 1), - 'compile_args': ('-Iusr/local/include/simple',), - 'link_args': ('-Lusr/local/lib/simple', '-lsimple') - }), - ("pot_hole", { - 'version': (3, 2, 1), - 'compile_args': ('-Iusr/local/include/pothole',), - 'link_args': ('-Lusr/local/lib/pothole', '-lpothole') - }), - ("CamelCase", { - 'version': (1, 2, 3), - 'compile_args': ('-Iopt/special/include/camel',), - 'link_args': ('-Lopt/special/lib/camel', '-lcamel') - }) - ]) - def test_constructor_name(self, name: str, expected: Dict[str, Any], - system_path: Path, system_pkg_path: Path, - user_path: Path, user_pkg_path: Path, - tmp_path: Path, monkeypatch: MonkeyPatch): - """ - Checks some likely library names. - - Libraries on both system and user paths. - """ - relative_path = system_path.relative_to(tmp_path) - (system_pkg_path / 'single.pc').write_text( - dedent( - f""" - Name: single - Version: 1.0.1 - Description: Single word, lower case. - URL: http://example.com/single - Cflags: -I{relative_path}/include/simple - Cflags.private: -I{relative_path}/include/static - Libs: -L{relative_path}/lib/simple -lsimple - Libs.private: -L{relative_path}/lib/static -l static - """ - ) - ) - - relative_path = system_path.relative_to(tmp_path) - (system_pkg_path / 'pot_hole.pc').write_text( - dedent( - f""" - Name: pot_hole - Version: 3.2.1 - Description: Two words, unlerine separated. - URL: http://example.com/pothole - Cflags: -I{relative_path}/include/pothole - Cflags.private: -I{relative_path}/include/static - Libs: -L{relative_path}/lib/pothole -lpothole - Libs.private: -L{relative_path}/lib/static -l static - """ - ) - ) - - relative_path = user_path.relative_to(tmp_path) - (user_pkg_path / 'CamelCase.pc').write_text( - dedent( - f""" - Name: CamelCase - Version: 1.2.3 - Description: Two words, initial capital. - URL: http://example.com/CamelCase - Cflags: -I{relative_path}/include/camel - Cflags.private: -I{relative_path}/include/static - Libs: -L{relative_path}/lib/camel -lcamel - Libs.private: -L{relative_path}/lib/static -l static - """ - ) - ) - - test_unit = Package(name) - assert test_unit.name == name - assert test_unit.version == expected['version'] - assert test_unit.compile_arguments == expected['compile_args'] - assert test_unit.link_arguments == expected['link_args'] - - @mark.parametrize('version, expected', [ - ('', { - 'version': tuple() - }), - ('1', { - 'version': (1,) - }), - ('1.2', { - 'version': (1, 2) - }), - ('1.2.3', { - 'version': (1, 2, 3) - }), - ('1.2.dev1', { - 'version': (1, 2, 'dev1') - }), - ('2.3.dev.2', { - 'version': (2, 3, 'dev', 2) - }) - ]) - def test_constructor_version(self, version: str, - expected: Dict[str, Any], - user_path: Path, user_pkg_path: Path, - tmp_path: Path): - """ - Checks various version number formats. - - Libraries on user paths. - """ - (user_pkg_path / 'test.pc').write_text( - dedent( - f""" - Name: test - Version: {version} - Description: Many versions. - URL: http://example.com/version - Cflags: -I{user_path.relative_to(tmp_path)}/include - Cflags.private: -I{user_path.relative_to(tmp_path)}/include - Libs: -L{user_path.relative_to(tmp_path)}/lib -lversion - Libs.private: -L{user_path.relative_to(tmp_path)}/lib -l static - """ - ) - ) - - test_unit = Package('test') - assert test_unit.name == 'test' - assert test_unit.version == expected['version'] - assert test_unit.compile_arguments == ('-Iopt/special/include',) - assert test_unit.link_arguments == ('-Lopt/special/lib', '-lversion') - - @mark.parametrize('arg_str, expected', [ - ('', tuple()), - ('-I/usr/local/lib/special', ('-I/usr/local/lib/special',)), - ('-I/usr/local/lib/special -I/usr/local/lib/other', - ('-I/usr/local/lib/special', '-I/usr/local/lib/other')), - ('-I /usr/local/special', ('-I/usr/local/special',)) - ]) - def test_constructor_compile_arguments(self, arg_str: str, - expected: Tuple[str], - user_path: Path, - user_pkg_path: Path, - tmp_path: Path): - """ - Checks compiler path arguments are compressed. - - Libraries on user paths. - """ - (user_pkg_path / 'test.pc').write_text( - dedent( - f""" - Name: test - Version: 1.0.0 - Description: Many versions. - URL: http://example.com/version - Cflags: {arg_str} - Cflags.private: - Libs: -L{user_path.relative_to(tmp_path)}/lib -lversion - Libs.private: -L{user_path.relative_to(tmp_path)}/lib -l static - """ - ) - ) - - test_unit = Package('test') - assert test_unit.name == 'test' - assert test_unit.version == (1, 0, 0) - assert test_unit.compile_arguments == expected - assert test_unit.link_arguments == ('-Lopt/special/lib', '-lversion') - - @mark.parametrize('arg_str, expected', [ - ('', tuple()), - ('-L/usr/local/lib/special -lspecial', - ('-L/usr/local/lib/special', '-lspecial')), - ('-L/usr/local/lib/special -lspecial -L/usr/local/lib/other -lother', - ('-L/usr/local/lib/special', '-lspecial', - '-L/usr/local/lib/other', '-lother')), - ('-L /usr/local/special -l special', - ('-L/usr/local/special', '-lspecial')) - ]) - def test_constructor_link_arguments(self, arg_str: str, - expected: Tuple[str], - user_path: Path, user_pkg_path: Path, - tmp_path: Path): - """ - Checks linker path arguments are compressed. - - Libraries on user paths. - """ - (user_pkg_path / 'test.pc').write_text( - dedent( - f""" - Name: test - Version: 1.0.0 - Description: Many versions. - URL: http://example.com/version - Cflags: -Iopt/special/include - Cflags.private: -I/opt/special/include - Libs: {arg_str} - Libs.private: - """ - ) - ) - - test_unit = Package('test') - assert test_unit.name == 'test' - assert test_unit.version == (1, 0, 0) - assert test_unit.compile_arguments == ('-Iopt/special/include',) - assert test_unit.link_arguments == expected - - @mark.parametrize('link_type, expected', [ - (LinkType.SHARED, '--shared'), (LinkType.STATIC, '--static') - ]) - def test_constructor_link_type(self, link_type: LinkType, expected: str, - user_path: Path, user_pkg_path: Path, - tmp_path: Path): - """ - Checks shared and static linking. - - Mock pkg_config acts as though library is on system path. - """ - (user_pkg_path / 'test.pc').write_text( - dedent( - """ - Name: test - Version: 1.0.0 - Description: Many versions. - URL: http://example.com/version - Cflags: -Iopt/special/include - Cflags.private: -I/opt/special/include/private - Libs: -Lopt/special/lib -l test - Libs.private: -L opt/special/lib/private -ltest-private - """ - ) - ) - - test_unit = Package('test', link_type=link_type) - assert test_unit.name == 'test' - assert test_unit.version == (1, 0, 0) - if link_type == LinkType.SHARED: - assert test_unit.compile_arguments == ('-Iopt/special/include',) - assert test_unit.link_arguments == ('-Lopt/special/lib', '-ltest') - elif link_type == LinkType.STATIC: - assert test_unit.compile_arguments == ( - '-Iopt/special/include', '-I/opt/special/include/private' - ) - assert test_unit.link_arguments == ( - '-Lopt/special/lib', '-ltest', - '-Lopt/special/lib/private', '-ltest-private' - ) - else: - assert False - - def test_constructor_tree(self, user_path: Path, user_pkg_path: Path, - system_path: Path, system_pkg_path: Path, - tmp_path: Path): - """ - Checks recursive dependencies. - """ - (system_pkg_path / 'system.pc').write_text( - dedent( - """ - Name: system - Version: 1.0.0 - Description: Is a system thing. - URL: http://example.com/system - Cflags: -Iusr/local/include - Libs: -Lusr/local/lib -lsystem - """ - ) - ) - - (user_pkg_path / 'user.pc').write_text( - dedent( - """ - Name: user - Version: 1.0.0 - Description: Depends on system thing. - URL: http://example.com/user - Requires: system - Cflags: -Iopt/user/include - Libs: -Lopt/user/lib -luser - """ - ) - ) - - test_unit = Package('user') - assert test_unit.name == 'user' - assert test_unit.version == (1, 0, 0) - assert test_unit.compile_arguments == ('-Iopt/user/include', - '-Iusr/local/include') - assert test_unit.link_arguments == ('-Lopt/user/lib', '-luser', - '-Lusr/local/lib', '-lsystem') - - @mark.parametrize('version, expected', - [ - ('0.1.0', None), ('1.2.2', None), - ('1.2.3', (1, 2, 3)), ('1.2.4', (1, 2, 4)) - ]) - def test_constructor_specification(self, version: str, - expected: Optional[Tuple[int]], - user_pkg_path: Path): - (user_pkg_path / 'test.pc').write_text( - dedent( - f""" - Name: test - Version: {version} - Description: Many versions. - URL: http://example.com/test - Cflags: -Iopt/test/include - Libs: -Lopt/test/lib -ltest - """ - ) - ) - if expected is None: - with raises(PackageException): - _ = Package('test >= 1.2.3') - else: # Package should fulfil specification - test_unit = Package('test >= 1.2.3') - assert test_unit.name == 'test' - assert test_unit.version == expected From 900a984d76d862fa46cfe3ed6475752e325b15b9 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 11:42:05 +1000 Subject: [PATCH 71/95] #292 Fixed typo in comment. --- lfric_build/lfric_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index eb666b46c..f432bd8aa 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -413,7 +413,7 @@ def get_transformation_script(self, fpath: Path, optimisation_path = (config.source_root / "optimisation" / f"{self.site}-{self.platform}" / "psykal") relative_path = None - # The soure file might be either in build_output (e.g. a preprocessed + # The source file might be either in build_output (e.g. a preprocessed # .X90 file), or still in source (.x90 file). Check if the file # is in one of the two sub-trees, and use the relative path to # check if there is a file-specific optimisation script From 3b397a6c414ed19d76ea10a7bbf9d5190d2d6e55 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 11:53:42 +1000 Subject: [PATCH 72/95] #292 Use instead of sys.path to add to python search path, since sys.path is not inherited by child processes (as which all tools run). --- lfric_build/lfric_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index f432bd8aa..2f67e38e1 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -14,6 +14,7 @@ import argparse import logging +import os from pathlib import Path import sys from typing import List, Optional, Iterable, Union @@ -375,15 +376,18 @@ def psyclone_step( psyclone_cli_args.extend(additional_parameters) # To avoid impacting other code, store the original search path - old_sys_path = sys.path[:] - sys.path.extend(self._add_python_paths) + orig_pythonpath = os.environ.get("PYTHONPATH", "") + add_python_paths = ":".join(str(i) for i in self._add_python_paths) + os.environ["PYTHONPATH"] = (f"{add_python_paths}:" + f"{orig_pythonpath}") psyclone(self.config, kernel_roots=[(self.config.build_output / "kernel")], transformation_script=self.get_transformation_script, api="lfric", cli_args=psyclone_cli_args, ignore_dependencies=ignore_dependencies) - sys.path = old_sys_path + # Reset PYTHONPATH + os.environ["PYTHONPATH"] = orig_pythonpath def get_psyclone_config(self) -> str: ''' From e33c642079d158a6c56e6821884f94fae7194c49 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 28 May 2026 18:29:27 +1000 Subject: [PATCH 73/95] #371 Better support running without a script. --- lfric_build/psyclone_control.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 2507b4085..98979e952 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -28,8 +28,14 @@ class PsycloneInfo: platform. """ + # Define a new 'script paths' to indicate special results. Use : in the + # name, which is typically not a valid name (at the start) + RESULT_EXCLUDE = Path("::RESULT_EXCLUDE") + RESULT_NO_SCRIPT = Path("::RESULT_NO_SCRIPT") + FILE_SPECIFIC = "file_specific" EXCLUDE = "exclude" + NO_SCRIPT = "no_script" def __init__(self, name: str, fab_base: FabBase) -> None: self._fab_base = fab_base @@ -167,7 +173,7 @@ def file_specific_script(self, fpath: Path) -> Optional[Path]: return local_transformation_script return None - def get_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: + def get_script(self, fpath: Path, config: BuildConfig) -> Path: """ This method returns the script to be used for a given filename, or None if no rule applies (or an explicit exclude rule applies) @@ -208,7 +214,11 @@ def get_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: elif rule == PsycloneInfo.EXCLUDE: # Exclude pattern matches: - return None + return PsycloneInfo.RESULT_EXCLUDE + + elif rule == PsycloneInfo.NO_SCRIPT: + # Exclude pattern matches: + return PsycloneInfo.RESULT_NO_SCRIPT else: opt_script = self.opt_path / rule @@ -217,7 +227,7 @@ def get_script(self, fpath: Path, config: BuildConfig) -> Optional[Path]: f"'{opt_script}'.") return opt_script - return None + return PsycloneInfo.RESULT_EXCLUDE class PsycloneControl: From cbaed2e29ab65f26595dbe74adbc1ca75d555f51 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 2 Jun 2026 12:41:29 +1000 Subject: [PATCH 74/95] #292 Updated unit tests. --- lfric_build/tests/lfric_base_test.py | 59 ++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 09c869ea3..459123655 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -163,20 +163,20 @@ def test_constructor(monkeypatch) -> None: Tests constructor. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test_name", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test_name", app_dir=Path(".")) # Check root symbol defaults to name if not specified assert lfric_base.root_symbol == ["test_name"] # Check root symbol can be specified lfric_base = LFRicBase(name="test_name", - apps_dir=Path("."), + app_dir=Path("."), root_symbol="root1") assert lfric_base.root_symbol == ["root1"] # Check root symbol list lfric_base = LFRicBase(name="test_name", - apps_dir=Path("."), + app_dir=Path("."), root_symbol=["root1", "root2"]) assert lfric_base.root_symbol == ["root1", "root2"] @@ -227,10 +227,11 @@ def create_frame_info(filename): monkeypatch.setattr('inspect.stack', lambda: mock_stack) monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=tmp_path / "app_dir") # Verify core root is set correctly assert lfric_base.lfric_core_root == mock_core + assert lfric_base.app_dir == tmp_path / "app_dir" def test_require_openmp(monkeypatch, caplog) -> None: @@ -242,7 +243,7 @@ def test_require_openmp(monkeypatch, caplog) -> None: "--no-openmp"]) with pytest.raises(SystemExit): - LFRicBase(name="test", apps_dir=Path(".")) + LFRicBase(name="test", app_dir=Path(".")) assert len(caplog.records) == 2 @@ -257,6 +258,24 @@ def test_require_openmp(monkeypatch, caplog) -> None: "the '-no-omp' flag from the command line." in caplog.text) +def test_netcdf_required(monkeypatch): + """ + Test that the class aborts if the linker does not have NetCDF defined. + """ + + tr = ToolRepository() + linker = tr.get_tool(Category.LINKER, "sln") + # The dummy linker includes netcdf - remove it: + monkeypatch.setattr(linker, "_lib_flags", {}) + + # Required, otherwise it would take pytest's command line as sys.argv + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + with pytest.raises(RuntimeError) as err: + LFRicBase(name="test", app_dir=Path(".")) + assert ("LFRic needs NetCDF, but the linker 'sln' has no NetCDF library " + "setting defined." in str(err)) + + def test_precision_definition_without_default(monkeypatch) -> None: ''' Tests specification of precision if no default precision is @@ -269,7 +288,7 @@ def test_precision_definition_without_default(monkeypatch) -> None: "--precision_other", "32"]) monkeypatch.setattr(os, 'environ', {"R_BL_PRECISION": "64"}) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) lfric_base.define_preprocessor_flags_step() flags = lfric_base.preprocess_flags_common @@ -302,7 +321,7 @@ def test_preprocessor_flags(monkeypatch, no_xios, mpi) -> None: fc = tr.get_tool(Category.FORTRAN_COMPILER, "sfc") monkeypatch.setattr(fc, "_mpi", mpi) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) lfric_base.define_preprocessor_flags_step() expected_flags = [ @@ -323,7 +342,7 @@ def test_setup_site_specific_location(monkeypatch) -> None: Tests site specific path setup for LFRicBase. ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) old_path = sys.path.copy() lfric_base.setup_site_specific_location() @@ -343,7 +362,7 @@ def test_get_linker_flags(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) flags = lfric_base.get_linker_flags() expected_libs = ['yaxt', 'xios', 'netcdf', 'hdf5'] @@ -363,7 +382,7 @@ def test_grab_files_step(monkeypatch) -> None: # Setup mocks monkeypatch.setattr('lfric_base.grab_folder', mock_grab) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) monkeypatch.setattr(lfric_base, '_lfric_core_root', mock_core) # Call method under test @@ -408,7 +427,7 @@ def test_find_source_files_step(monkeypatch) -> None: with (mock.patch('lfric_base.FabBase.find_source_files_step') as find_step, mock.patch('lfric_base.LFRicBase.templaterator_step') as temp_step, mock.patch('lfric_base.LFRicBase.configurator_step') as conf_step): - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) lfric_base.find_source_files_step() # Verify super called @@ -431,7 +450,7 @@ def test_configurator_step(monkeypatch) -> None: # Set up mocks using monkeypatch monkeypatch.setattr('lfric_base.configurator', mock_config) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) monkeypatch.setattr(lfric_base, 'get_rose_meta', mock_meta) with pytest.warns(match="_metric_send_conn not set, cannot send metrics"): @@ -483,7 +502,7 @@ def test_templaterator_step(monkeypatch, tmp_path) -> None: monkeypatch.setattr('lfric_base.SuffixFilter', lambda *args: mock_filter) # Create LFRicBase instance - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) monkeypatch.setattr(lfric_base, '_lfric_core_root', tmp_path) # Run templaterator step @@ -538,7 +557,7 @@ def test_get_rose_meta(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) assert lfric_base.get_rose_meta() is None @@ -557,7 +576,7 @@ def test_analyse_step(monkeypatch) -> None: monkeypatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', mock_analyse) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) # Mock instance methods monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) @@ -588,7 +607,7 @@ def test_analyse_step(monkeypatch) -> None: mock_preprocess.reset_mock() mock_psyclone.reset_mock() - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) @@ -618,7 +637,7 @@ def test_preprocess_x90_step(monkeypatch) -> None: mock_preproc = mock.MagicMock() monkeypatch.setattr('lfric_base.preprocess_x90', mock_preproc) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) lfric_base.preprocess_x90_step() @@ -641,7 +660,7 @@ def test_psyclone_step(monkeypatch) -> None: # Set up monkeypatch for module level import monkeypatch.setattr('lfric_base.psyclone', mock_psy) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) # Patch instance methods. Return a copy to avoid that # PSyclone modified these lists in the lambdas when it modifies the list @@ -668,7 +687,7 @@ def test_get_psyclone_config(monkeypatch) -> None: ''' monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) config_args = lfric_base.get_psyclone_config() assert config_args == str(lfric_base.config.source_root / @@ -682,7 +701,7 @@ def test_get_transformation_script(monkeypatch, tmp_path) -> None: monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) # Create LFRicBase instance with mocked site/platform - lfric_base = LFRicBase(name="test", apps_dir=Path(".")) + lfric_base = LFRicBase(name="test", app_dir=Path(".")) # Create mock config config = mock.MagicMock() From c14d2432ce43deffe0d978eaf73dbd65f85be6a1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 2 Jun 2026 14:12:44 +1000 Subject: [PATCH 75/95] #371 Fixed failing nf_config test. --- lfric_build/tests/nf_config_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lfric_build/tests/nf_config_test.py b/lfric_build/tests/nf_config_test.py index 464796f8a..7d6f6a1b9 100644 --- a/lfric_build/tests/nf_config_test.py +++ b/lfric_build/tests/nf_config_test.py @@ -33,7 +33,6 @@ def test_constructor() -> None: assert nfc.category == Category.MISC assert nfc.name == "nf-config" assert nfc.exec_name == "nf-config" - assert nfc.get_flags() == [] def test_nf_config_check_available(fake_process: FakeProcess) -> None: From acf57a66dec7a2349e9928618a7f4df7fc8a5cd3 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 2 Jun 2026 14:27:16 +1000 Subject: [PATCH 76/95] #371 Updated tests for new psyclone usage. --- lfric_build/tests/lfric_base_test.py | 58 +++++++++++++++++++--------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 459123655..e275d12d0 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -226,6 +226,8 @@ def create_frame_info(filename): ] monkeypatch.setattr('inspect.stack', lambda: mock_stack) monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + psyclone_control = mock_base_dir / "psyclone_info.yaml" + psyclone_control.write_text("phases:", encoding='utf-8') lfric_base = LFRicBase(name="test", app_dir=tmp_path / "app_dir") @@ -691,7 +693,7 @@ def test_get_psyclone_config(monkeypatch) -> None: config_args = lfric_base.get_psyclone_config() assert config_args == str(lfric_base.config.source_root / - 'psyclone_config/psyclone.cfg') + 'psyclone_config' / 'psyclone.cfg') def test_get_transformation_script(monkeypatch, tmp_path) -> None: @@ -703,47 +705,67 @@ def test_get_transformation_script(monkeypatch, tmp_path) -> None: # Create LFRicBase instance with mocked site/platform lfric_base = LFRicBase(name="test", app_dir=Path(".")) - # Create mock config + # Create mock config, and insert it into the lfric base instance + # (to mock the paths used here) config = mock.MagicMock() config.source_root = tmp_path - config.build_output = tmp_path / "build" + config._build_output = tmp_path / "build" config.build_output.mkdir() + lfric_base._config = config + + optimisation_folder_path = (tmp_path / "optimisation" / "default-default" / + "psykal") + + # Set a current psyclone phase in the object, so the correct rule + # (for dsl) is picked: + psy_info = lfric_base._psyclone_control.get_info("dsl") + psy_info._opt_path = optimisation_folder_path + lfric_base._current_psyclone_info = psy_info # Create x90 test source file - source_path = tmp_path / "some/path" + source_path = tmp_path / "some" / "path" source_path.mkdir(parents=True) + + # Test case 1: No optimisation directory, no transformation script test_file = source_path / "file.x90" test_file.touch() + with pytest.raises(FileNotFoundError) as err: + lfric_base.get_transformation_script(test_file, config) - # Test case 1: x90 file not in source or build directories - outside_file = tmp_path.parent / "outside.x90" - assert lfric_base.get_transformation_script(outside_file, config) is None - - # Test case 2: No optimisation directory, no transformation script - assert lfric_base.get_transformation_script(test_file, config) is None - - # Test case 3: No PSykal but optimisation directory - optimisation_folder_path = (tmp_path / "optimisation" / "default-default" / - "psykal") + # Test case 2: No PSykal but optimisation directory global_script = optimisation_folder_path / "global.py" global_script.parent.mkdir(parents=True) global_script.touch() # No file-specific transformation script, use global script - other_file = tmp_path / "other/path/test.x90" + other_file = tmp_path / "other" / "path" / "test.x90" other_file.parent.mkdir(parents=True) other_file.touch() assert (lfric_base.get_transformation_script(other_file, config) == global_script) - # Test case 4: Psykal directory exists - psykal_path = tmp_path / "optimisation/default-default/psykal" + # Test case 3: Psykal directory exists + psykal_path = tmp_path / "optimisation" / "default-default" / "psykal" # Create specific transformation script in psykal dir - specific_script = psykal_path / "some/path/file.py" + specific_script = psykal_path / "some" / "path" / "file.py" specific_script.parent.mkdir(parents=True) specific_script.touch() # Use specific script in psykal directory assert lfric_base.get_transformation_script(test_file, config) == \ specific_script + + # Test case 4: Return exclude (which should not happen, the function + # should only be called for files that have a script) + psy_info._rules = [] + with pytest.raises(ValueError) as err: + lfric_base.get_transformation_script(test_file, config) + assert ("PSyclone transformation script returned " + "'PsycloneInfo.RESULT_EXCLUDE', which should not happen." + in str(err)) + + # Test case 5: Return None if the PSyclone control file explicitly + # requests to run PSyclone without a script. + psy_info._rules = [(psy_info.NO_SCRIPT, "*")] + assert lfric_base.get_transformation_script(test_file, config) is None From 236ad57cafbe60fe0869cdb1a6887d591ea69b5f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 2 Jun 2026 14:27:40 +1000 Subject: [PATCH 77/95] #371 Linting changes. --- lfric_build/psyclone_control.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 98979e952..26cb51fb4 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -212,20 +212,19 @@ def get_script(self, fpath: Path, config: BuildConfig) -> Path: raise FileNotFoundError( f"Cannot find explicitly requested script '{script}'.") - elif rule == PsycloneInfo.EXCLUDE: + if rule == PsycloneInfo.EXCLUDE: # Exclude pattern matches: return PsycloneInfo.RESULT_EXCLUDE - elif rule == PsycloneInfo.NO_SCRIPT: + if rule == PsycloneInfo.NO_SCRIPT: # Exclude pattern matches: return PsycloneInfo.RESULT_NO_SCRIPT - else: - opt_script = self.opt_path / rule - if not opt_script.exists(): - raise FileExistsError(f"Cannot find script " - f"'{opt_script}'.") - return opt_script + opt_script = self.opt_path / rule + if not opt_script.exists(): + raise FileNotFoundError(f"Cannot find script " + f"'{opt_script}'.") + return opt_script return PsycloneInfo.RESULT_EXCLUDE From 4b1ee14d1f7f4040e1ed2d6d3c069da62e17ba3d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 3 Jun 2026 22:42:28 +1000 Subject: [PATCH 78/95] #371 Removed Fab as dependency of psyclone_control.py. --- lfric_build/lfric_base.py | 27 +++++++++++++++-------- lfric_build/psyclone_control.py | 39 ++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 47403f5e0..9945405ff 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -87,8 +87,16 @@ def __init__(self, name: str, f"has no NetCDF library setting defined. Aborting.") raise RuntimeError(msg) from err - # Stores the PSyclone control information - self._psyclone_control = PsycloneControl(self) + # Store the PSyclone control information + # The paths used when identifying file-specific scripts. The matching + # base path is removed from the file name to get a relative name + # which is used in matching. + base_paths = [self.config.source_root, + self.config.build_output] + script_root = (self.config.source_root / "optimisation" / + f"{self.site}-{self.platform}") + self._psyclone_control = PsycloneControl(script_root=script_root, + base_paths=base_paths) # If the PSyclone step is running, this stores the currently # executed PSyclone information. This is used in the @@ -409,16 +417,17 @@ def psyclone_step( psyclone_cli_args.extend(additional_parameters) add_python_paths = ":".join(str(i) for i in self._add_python_paths) + # To avoid impacting other code, store the original search path + # We have to modify PYTHONPATH (and not sys.path), since PSyclone + # is run in its own shell (i.e. it inherits PYTHONPATH, but not + # sys.path). + orig_pythonpath = os.environ.get("PYTHONPATH", "") + for phase in self._psyclone_control.all_phases: logger.info(f"Running PSyclone phase {phase}.") psyclone_info = self._psyclone_control.get_info(phase) # Save the info for the get transformation script function self._current_psyclone_info = psyclone_info - # To avoid impacting other code, store the original search path - # We have to modify PYTHONPATH (and not sys.path), since PSyclone - # is run in its own shell (i.e. it inherits PYTHONPATH, but not - # sys.path). - orig_pythonpath = os.environ.get("PYTHONPATH", "") # Add various paths: optimisation/site-platform/transmute # (=opt_path) is required for some transmute scripts that # import helper functions. Adding add_python_paths is @@ -449,7 +458,7 @@ def _psyclone_transmute(self, f90_files: list[Path] = [] af_store = self.config.artefact_store for file in af_store[ArtefactSet.FORTRAN_COMPILER_FILES]: - script = psyclone_info.get_script(file, self.config) + script = psyclone_info.get_script(file) if script != PsycloneInfo.RESULT_EXCLUDE: f90_files.append(file) @@ -492,7 +501,7 @@ def get_transformation_script(self, fpath: Path, :raises ValueError: If the current psyclone_info result indicates that the current file should be be run through PSyclone. ''' - script = self._current_psyclone_info.get_script(fpath, config) + script = self._current_psyclone_info.get_script(fpath) if script == PsycloneInfo.RESULT_NO_SCRIPT: return None diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 26cb51fb4..d48937037 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -13,9 +13,6 @@ from typing import Optional, Union import yaml -from fab.api import BuildConfig -from fab.fab_base.fab_base import FabBase - class PsycloneInfo: """ @@ -37,11 +34,14 @@ class PsycloneInfo: EXCLUDE = "exclude" NO_SCRIPT = "no_script" - def __init__(self, name: str, fab_base: FabBase) -> None: - self._fab_base = fab_base + def __init__(self, name: str, + base_paths: list[Path], + script_root: Path) -> None: + self._base_paths = base_paths + self._script_root = script_root # This will be initialised/updated each time when reading an info file. self._opt_path = Path() - self._script_dir: str = "" + self._relative_script_dir: str = "" self._name: str = name self._comment: str = "" self._api: str = "" @@ -102,16 +102,14 @@ def update(self, info: dict[str, str]) -> None: elif rule == "artefacts": self._artefacts = info["artefacts"] elif rule == "script_dir": - self._script_dir = info["script_dir"] + self._relative_script_dir = info["script_dir"] else: self._read_rule(rule, info[rule]) # Store the potentially updated optimisation root path, i.e. the site- # and platform-specific location, followed by a script dir (typically # transmute or psykal). This path is used in a few places. - self._opt_path = (self._fab_base.config.source_root / "optimisation" / - f"{self._fab_base.site}-{self._fab_base.platform}" / - self._script_dir) + self._opt_path = self._script_root / self._relative_script_dir def _read_rule(self, rule: str, file_list: str) -> None: """ @@ -136,7 +134,7 @@ def view(self) -> str: comment: {self.comment} api: {self.api} artefacts: {self.artefacts} -script_dir: {self._script_dir} +script_dir: {self._relative_script_dir} rules: {self._rules} """ return s @@ -156,8 +154,7 @@ def file_specific_script(self, fpath: Path) -> Optional[Path]: # .X90 file), or still in source (.x90 file). Check if the file # is in one of the two sub-trees, and use the relative path to # check if there is a file-specific optimisation script - for base_path in [self._fab_base.config.source_root, - self._fab_base.config.build_output]: + for base_path in self._base_paths: try: relative_path = fpath.relative_to(base_path) except ValueError: @@ -173,7 +170,7 @@ def file_specific_script(self, fpath: Path) -> Optional[Path]: return local_transformation_script return None - def get_script(self, fpath: Path, config: BuildConfig) -> Path: + def get_script(self, fpath: Path) -> Path: """ This method returns the script to be used for a given filename, or None if no rule applies (or an explicit exclude rule applies) @@ -183,7 +180,6 @@ def get_script(self, fpath: Path, config: BuildConfig) -> Path: :param fpath: the Fortran source file for which to find a transformation script. - :param config: the build configuration (unused in this implementation) :returns: the path to the transformation script, or None if no rule applies (or an exclude rule applies). @@ -237,13 +233,18 @@ class PsycloneControl: Details of each phase will be stored in PsycloneInfo instances. + :param base_paths: :param fab_base: The FabBase derived application script. This is required to get site, platform and config information when searching for PSyclone scripts to be executed. """ - def __init__(self, fab_base: FabBase) -> None: - self._fab_base = fab_base + def __init__(self, + script_root: Path, + base_paths: list[Path]) -> None: + # Keep a copy in case that the user modifies the list later + self._base_paths = base_paths[:] + self._script_root = script_root self._all_phases: list[str] = [] self._psyclone_info: dict[str, PsycloneInfo] = {} @@ -302,6 +303,8 @@ def read(self, filename: Union[str, Path]) -> None: # Already handled continue if key not in self._psyclone_info: - self._psyclone_info[key] = PsycloneInfo(key, self._fab_base) + self._psyclone_info[key] = PsycloneInfo(key, + self._base_paths, + self._script_root) self._psyclone_info[key].update(dependencies[key]) From ab8ed85b76cf1e6f3f18a3162292dd0d11b836e8 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 10 Jun 2026 12:47:54 +1000 Subject: [PATCH 79/95] #371 Small NCI bug fix. --- lfric_build/site_specific/nci_gadi/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lfric_build/site_specific/nci_gadi/config.py b/lfric_build/site_specific/nci_gadi/config.py index 282a322e7..c637d0d4e 100644 --- a/lfric_build/site_specific/nci_gadi/config.py +++ b/lfric_build/site_specific/nci_gadi/config.py @@ -17,6 +17,7 @@ from fab.api import BuildConfig, Category, Linker, ToolRepository from default.config import Config as DefaultConfig +from nf_config import NfConfig class Config(DefaultConfig): From ba03118203d9633ce5e79c92c1a8bdbe5c588091 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 10 Jun 2026 13:19:11 +1000 Subject: [PATCH 80/95] #371 Add the PSyclone control file to the logger. --- lfric_build/lfric_base.py | 12 ++++----- lfric_build/psyclone_control.py | 48 ++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 9aea99521..958023185 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -28,13 +28,8 @@ from templaterator import Templaterator from psyclone_control import PsycloneControl, PsycloneInfo -# Add a logger and connect it to stdout. logger = logging.getLogger("fab") -# Add a logger and connect it to stdout. -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) - class LFRicBase(FabBase): ''' @@ -415,6 +410,11 @@ def psyclone_step( Folders containing kernel files. Must be part of the analysed source code. ''' + logger.info(f"PSyclone control file\n" + f"# -------------------\n" + f"{self._psyclone_control.to_yaml()}\n" + f"# -------------------\n") + kernel_roots = kernel_roots or [] psyclone_cli_args = ["--config", self.get_psyclone_config()] if additional_parameters: @@ -428,8 +428,8 @@ def psyclone_step( orig_pythonpath = os.environ.get("PYTHONPATH", "") for phase in self._psyclone_control.all_phases: - logger.info(f"Running PSyclone phase {phase}.") psyclone_info = self._psyclone_control.get_info(phase) + logger.info(f"Running PSyclone phase: {psyclone_info.comment}.") # Save the info for the get transformation script function self._current_psyclone_info = psyclone_info # Add various paths: optimisation/site-platform/transmute diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index d48937037..655b07ca5 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -10,7 +10,7 @@ This module reads in a psyclone_info.yaml file. ''' from pathlib import Path -from typing import Optional, Union +from typing import Optional import yaml @@ -126,18 +126,21 @@ def _read_rule(self, rule: str, file_list: str) -> None: file_list = "*" self._rules.append((rule, file_list.split())) - def view(self) -> str: + def get_yaml_dict(self) -> dict[str, list[str]]: """ - :returns: a string representation of this phase in yaml format. + :returns: returns ths data in this object in a dictionary, suitable + to write them back as a yaml file. """ - s = f"""{self._name}: -comment: {self.comment} -api: {self.api} -artefacts: {self.artefacts} -script_dir: {self._relative_script_dir} -rules: {self._rules} -""" - return s + + yaml_dict = {"comment": self.comment, + "api": self.api, + "artefacts": self.artefacts, + "script_dir": self._relative_script_dir} + + for rule, file_list in self._rules: + yaml_dict[rule] = " ".join(file_list) + + return yaml_dict def file_specific_script(self, fpath: Path) -> Optional[Path]: """ @@ -247,6 +250,9 @@ def __init__(self, self._script_root = script_root self._all_phases: list[str] = [] self._psyclone_info: dict[str, PsycloneInfo] = {} + # A list of all PSyclone info files that were read. + # Only used to add useful comments to the yaml output. + self._all_files_read: list[Path] = [] @property def all_phases(self) -> list[str]: @@ -265,7 +271,7 @@ def get_info(self, phase: str) -> PsycloneInfo: """ return self._psyclone_info[phase] - def view(self) -> str: + def to_yaml(self) -> str: """ This returns a string representation of the combined read yaml files. This is useful to be logged to show the actual details used. @@ -273,12 +279,17 @@ def view(self) -> str: :returns: the string represenation in yaml format of this PSyclone control instance. """ - s = f"""Phases: {" ".join(self._all_phases)}\n\n""" + + yaml_dict = {"phases": self._all_phases} for phase in self._all_phases: - s += f"{self._psyclone_info[phase].view()}\n" - return s + yaml_dict[phase] = self._psyclone_info[phase].get_yaml_dict() + + files_read = '\n'.join(f"# {i}" for i in self._all_files_read) + yaml_string = yaml.dump(yaml_dict, default_flow_style=False, + sort_keys=False) + return f"# Files read:\n{files_read}\n{yaml_string}" - def read(self, filename: Union[str, Path]) -> None: + def read(self, file_path: Path) -> None: """ Reads a yaml file, and extends the potentially existing information. Any phases specified in the new read yaml file will replace the @@ -288,11 +299,12 @@ def read(self, filename: Union[str, Path]) -> None: existing information. The precedence handling means that any later rule will overwrite any previous rule. - :param filename: the filename to read. + :param file_path: the file_path to read. """ - with open(filename, "r", encoding="utf8") as stream: + with open(file_path, "r", encoding="utf8") as stream: dependencies = yaml.safe_load(stream) + self._all_files_read.append(file_path.resolve()) # First take phases (if available) if dependencies.get("phases", None): From 2ab9ed8205ad7cd4c21b2ef07de7f050613129b1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 10 Jun 2026 17:19:06 +1000 Subject: [PATCH 81/95] #371 Added gemini-created unit test. --- lfric_build/tests/psyclone_control_test.py | 180 +++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 lfric_build/tests/psyclone_control_test.py diff --git a/lfric_build/tests/psyclone_control_test.py b/lfric_build/tests/psyclone_control_test.py new file mode 100644 index 000000000..96fd2a25d --- /dev/null +++ b/lfric_build/tests/psyclone_control_test.py @@ -0,0 +1,180 @@ +# ############################################################################ +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +# ############################################################################ + +''' +Unit tests for the psyclone_control module. +''' + +import pytest +from pathlib import Path +import yaml + +from psyclone_control import PsycloneInfo, PsycloneControl + + +def test_psyclone_info_properties(): + """Test initial properties and getters of PsycloneInfo.""" + base_paths = [Path("/base1"), Path("/base2")] + script_root = Path("/scripts") + + info = PsycloneInfo(name="test_phase", base_paths=base_paths, script_root=script_root) + + assert info.name == "test_phase" + assert info.comment == "" + assert info.api == "" + assert info.artefacts == "" + assert info.opt_path == Path() + + +@pytest.mark.parametrize("escaped_wildcard", ["\\*", "'*'", '"*"']) +def test_psyclone_info_wildcard_handling(escaped_wildcard): + """Verify different yaml wildcard string variants parse as expected.""" + info = PsycloneInfo("phase", [], Path()) + # Indirectly hit _read_rule via update + yaml_dict = {"some_script.py": escaped_wildcard} + info.update(yaml_dict) + + assert info._rules == [("some_script.py", ["*"])] + + +def test_psyclone_control_read_and_to_yaml(tmp_path): + """Test full workflow: reading configuration files and generating YAML output.""" + yaml_content_1 = """ +phases: + - dsl +dsl: + comment: "PSyclone DSL Phase" + api: lfric + artefacts: x90 + script_dir: psykal + global.py: \\* + file_specific: \\* +""" + yaml_file_1 = tmp_path / "psyclone_info.yaml" + yaml_file_1.write_text(yaml_content_1, encoding="utf-8") + + # Second YAML payload tests appending/overwriting phase specifications + yaml_content_2 = """ +phases: + - dsl + - secondary +secondary: + comment: "Secondary phase" + script_dir: alternative +""" + yaml_file_2 = tmp_path / "psyclone_info_override.yaml" + yaml_file_2.write_text(yaml_content_2, encoding="utf-8") + + base_paths = [tmp_path / "src", tmp_path / "build"] + script_root = tmp_path / "scripts" + + pc = PsycloneControl(script_root=script_root, base_paths=base_paths) + + # Read first file + pc.read(yaml_file_1) + assert pc.all_phases == ["dsl"] + + info_dsl = pc.get_info("dsl") + assert info_dsl.comment == "PSyclone DSL Phase" + assert info_dsl.api == "lfric" + assert info_dsl.artefacts == "x90" + assert info_dsl.opt_path == script_root / "psykal" + + # Read second file to test incremental overrides + pc.read(yaml_file_2) + assert pc.all_phases == ["dsl", "secondary"] + assert pc.get_info("secondary").comment == "Secondary phase" + + # Test YAML text generation output matches structures + yaml_out = pc.to_yaml() + assert f"# {yaml_file_1.resolve()}" in yaml_out + assert f"# {yaml_file_2.resolve()}" in yaml_out + + parsed_out = yaml.safe_load(yaml_out) + assert parsed_out["phases"] == ["dsl", "secondary"] + assert parsed_out["dsl"]["api"] == "lfric" + + +def test_file_specific_script_resolution(tmp_path): + """Validate looking up file-specific scripts inside source tree trees.""" + base_src = tmp_path / "src" + base_build = tmp_path / "build" + script_root = tmp_path / "scripts" + + # Define an active script path destination directory + opt_dir = script_root / "psykal" + opt_dir.mkdir(parents=True) + + info = PsycloneInfo(name="dsl", base_paths=[base_src, base_build], script_root=script_root) + info.update({"script_dir": "psykal"}) + + # Case 1: Target file is out of any base path boundary + external_file = tmp_path / "outside" / "some_mod.x90" + assert info.file_specific_script(external_file) is None + + # Case 2: Target inside src directory, but no python script companion exists yet + src_file = base_src / "kernel" / "some_mod.x90" + assert info.file_specific_script(src_file) is None + + # Case 3: Script target exists matching the relative structure + expected_script = opt_dir / "kernel" / "some_mod.py" + expected_script.parent.mkdir(parents=True, exist_ok=True) + expected_script.touch() + + assert info.file_specific_script(src_file) == expected_script + + +def test_get_script_matching_logic(tmp_path): + """Verify filtering behavior, fallback hierarchies, and exception conditions.""" + base_src = tmp_path / "src" + script_root = tmp_path / "scripts" + opt_dir = script_root / "psykal" + opt_dir.mkdir(parents=True) + + info = PsycloneInfo(name="dsl", base_paths=[base_src], script_root=script_root) + + # Setup rules: non-matching pattern, NO_SCRIPT rule, then explicit scripts + yaml_config = { + "script_dir": "psykal", + "exclude": "ignored_module.x90", + "no_script": "skipped_module.x90", + "missing_script.py": "broken_module.x90", + "valid_script.py": "good_module.x90", + "file_specific": "\\*", + } + info.update(yaml_config) + + # 1. Test standard pattern mismatch fallback + assert info.get_script(Path("unmatched_file.x90")) == PsycloneInfo.RESULT_EXCLUDE + + # 2. Test explicit EXCLUDE rules matching + assert info.get_script(Path("ignored_module.x90")) == PsycloneInfo.RESULT_EXCLUDE + + # 3. Test explicit NO_SCRIPT matching + assert info.get_script(Path("skipped_module.x90")) == PsycloneInfo.RESULT_NO_SCRIPT + + # 4. Test explicit rule script missing physically on storage disk + with pytest.raises(FileNotFoundError, match="Cannot find script '.*missing_script.py'"): + info.get_script(Path("broken_module.x90")) + + # 5. Test explicit rule script that exists successfully + valid_script_path = opt_dir / "valid_script.py" + valid_script_path.touch() + assert info.get_script(Path("good_module.x90")) == valid_script_path + + # 6. Test file_specific script execution with non-existent explicit definition + info_strict = PsycloneInfo(name="dsl", base_paths=[base_src], script_root=script_root) + info_strict.update({ + "script_dir": "psykal", + "file_specific": "explicit_custom.x90" + }) + with pytest.raises(FileNotFoundError, match="Cannot find explicitly requested script"): + info_strict.get_script(base_src / "explicit_custom.x90") + + # 7. File_specific if the script exists: + valid_script_path = opt_dir / "file_specific.py" + valid_script_path.touch() + assert info.get_script(base_src / "file_specific.x90") == valid_script_path From 6135b1240ff604c23093cdd9e9463267de6578eb Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 10 Jun 2026 17:23:45 +1000 Subject: [PATCH 82/95] #371 Black formatted, fixed line length. --- lfric_build/tests/psyclone_control_test.py | 100 +++++++++++++-------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/lfric_build/tests/psyclone_control_test.py b/lfric_build/tests/psyclone_control_test.py index 96fd2a25d..e32bb336e 100644 --- a/lfric_build/tests/psyclone_control_test.py +++ b/lfric_build/tests/psyclone_control_test.py @@ -4,24 +4,28 @@ # which you should have received as part of this distribution # ############################################################################ -''' +""" Unit tests for the psyclone_control module. -''' +""" -import pytest from pathlib import Path +import pytest import yaml from psyclone_control import PsycloneInfo, PsycloneControl def test_psyclone_info_properties(): - """Test initial properties and getters of PsycloneInfo.""" + """ + Test initial properties and getters of PsycloneInfo. + """ base_paths = [Path("/base1"), Path("/base2")] script_root = Path("/scripts") - - info = PsycloneInfo(name="test_phase", base_paths=base_paths, script_root=script_root) - + + info = PsycloneInfo( + name="test_phase", base_paths=base_paths, script_root=script_root + ) + assert info.name == "test_phase" assert info.comment == "" assert info.api == "" @@ -31,17 +35,21 @@ def test_psyclone_info_properties(): @pytest.mark.parametrize("escaped_wildcard", ["\\*", "'*'", '"*"']) def test_psyclone_info_wildcard_handling(escaped_wildcard): - """Verify different yaml wildcard string variants parse as expected.""" + """ + Verify different yaml wildcard string variants parse as expected. + """ info = PsycloneInfo("phase", [], Path()) # Indirectly hit _read_rule via update yaml_dict = {"some_script.py": escaped_wildcard} info.update(yaml_dict) - + assert info._rules == [("some_script.py", ["*"])] def test_psyclone_control_read_and_to_yaml(tmp_path): - """Test full workflow: reading configuration files and generating YAML output.""" + """ + Test full workflow: reading configuration files and generating YAML output. + """ yaml_content_1 = """ phases: - dsl @@ -70,13 +78,13 @@ def test_psyclone_control_read_and_to_yaml(tmp_path): base_paths = [tmp_path / "src", tmp_path / "build"] script_root = tmp_path / "scripts" - + pc = PsycloneControl(script_root=script_root, base_paths=base_paths) - + # Read first file pc.read(yaml_file_1) assert pc.all_phases == ["dsl"] - + info_dsl = pc.get_info("dsl") assert info_dsl.comment == "PSyclone DSL Phase" assert info_dsl.api == "lfric" @@ -92,30 +100,34 @@ def test_psyclone_control_read_and_to_yaml(tmp_path): yaml_out = pc.to_yaml() assert f"# {yaml_file_1.resolve()}" in yaml_out assert f"# {yaml_file_2.resolve()}" in yaml_out - + parsed_out = yaml.safe_load(yaml_out) assert parsed_out["phases"] == ["dsl", "secondary"] assert parsed_out["dsl"]["api"] == "lfric" def test_file_specific_script_resolution(tmp_path): - """Validate looking up file-specific scripts inside source tree trees.""" + """ + Validate looking up file-specific scripts inside source tree trees. + """ base_src = tmp_path / "src" base_build = tmp_path / "build" script_root = tmp_path / "scripts" - - # Define an active script path destination directory + + # Define an active script path destination directory opt_dir = script_root / "psykal" opt_dir.mkdir(parents=True) - - info = PsycloneInfo(name="dsl", base_paths=[base_src, base_build], script_root=script_root) + + info = PsycloneInfo( + name="dsl", base_paths=[base_src, base_build], script_root=script_root + ) info.update({"script_dir": "psykal"}) # Case 1: Target file is out of any base path boundary external_file = tmp_path / "outside" / "some_mod.x90" assert info.file_specific_script(external_file) is None - # Case 2: Target inside src directory, but no python script companion exists yet + # Case 2: Target inside src directory, but no companion exists src_file = base_src / "kernel" / "some_mod.x90" assert info.file_specific_script(src_file) is None @@ -123,19 +135,23 @@ def test_file_specific_script_resolution(tmp_path): expected_script = opt_dir / "kernel" / "some_mod.py" expected_script.parent.mkdir(parents=True, exist_ok=True) expected_script.touch() - + assert info.file_specific_script(src_file) == expected_script def test_get_script_matching_logic(tmp_path): - """Verify filtering behavior, fallback hierarchies, and exception conditions.""" + """ + Verify filtering behavior, fallback hierarchies, and exception conditions. + """ base_src = tmp_path / "src" script_root = tmp_path / "scripts" opt_dir = script_root / "psykal" opt_dir.mkdir(parents=True) - info = PsycloneInfo(name="dsl", base_paths=[base_src], script_root=script_root) - + info = PsycloneInfo( + name="dsl", base_paths=[base_src], script_root=script_root + ) + # Setup rules: non-matching pattern, NO_SCRIPT rule, then explicit scripts yaml_config = { "script_dir": "psykal", @@ -148,16 +164,27 @@ def test_get_script_matching_logic(tmp_path): info.update(yaml_config) # 1. Test standard pattern mismatch fallback - assert info.get_script(Path("unmatched_file.x90")) == PsycloneInfo.RESULT_EXCLUDE + assert ( + info.get_script(Path("unmatched_file.x90")) + == PsycloneInfo.RESULT_EXCLUDE + ) # 2. Test explicit EXCLUDE rules matching - assert info.get_script(Path("ignored_module.x90")) == PsycloneInfo.RESULT_EXCLUDE + assert ( + info.get_script(Path("ignored_module.x90")) + == PsycloneInfo.RESULT_EXCLUDE + ) # 3. Test explicit NO_SCRIPT matching - assert info.get_script(Path("skipped_module.x90")) == PsycloneInfo.RESULT_NO_SCRIPT + assert ( + info.get_script(Path("skipped_module.x90")) + == PsycloneInfo.RESULT_NO_SCRIPT + ) # 4. Test explicit rule script missing physically on storage disk - with pytest.raises(FileNotFoundError, match="Cannot find script '.*missing_script.py'"): + with pytest.raises( + FileNotFoundError, match="Cannot find script '.*missing_script.py'" + ): info.get_script(Path("broken_module.x90")) # 5. Test explicit rule script that exists successfully @@ -165,13 +192,16 @@ def test_get_script_matching_logic(tmp_path): valid_script_path.touch() assert info.get_script(Path("good_module.x90")) == valid_script_path - # 6. Test file_specific script execution with non-existent explicit definition - info_strict = PsycloneInfo(name="dsl", base_paths=[base_src], script_root=script_root) - info_strict.update({ - "script_dir": "psykal", - "file_specific": "explicit_custom.x90" - }) - with pytest.raises(FileNotFoundError, match="Cannot find explicitly requested script"): + # 6. Test file_specific script with non-existent explicit definition + info_strict = PsycloneInfo( + name="dsl", base_paths=[base_src], script_root=script_root + ) + info_strict.update( + {"script_dir": "psykal", "file_specific": "explicit_custom.x90"} + ) + with pytest.raises( + FileNotFoundError, match="Cannot find explicitly requested script" + ): info_strict.get_script(base_src / "explicit_custom.x90") # 7. File_specific if the script exists: From c8138deb5a9d4a67a3407a12e60b56f9bdc06bcd Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 10 Jun 2026 17:36:52 +1000 Subject: [PATCH 83/95] #371 Removed artefacts attribute, which is not used or required. --- lfric_build/psyclone_control.py | 11 ----------- lfric_build/tests/psyclone_control_test.py | 3 --- 2 files changed, 14 deletions(-) diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 655b07ca5..51b2a2cab 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -45,7 +45,6 @@ def __init__(self, name: str, self._name: str = name self._comment: str = "" self._api: str = "" - self._artefacts: str = "" self._rules: list[tuple[str, list[str]]] = [] @property @@ -69,13 +68,6 @@ def api(self) -> str: """ return self._api - @property - def artefacts(self) -> str: - """ - :returns: the artefacts to apply this info to. - """ - return self._artefacts - @property def opt_path(self) -> Path: """ @@ -99,8 +91,6 @@ def update(self, info: dict[str, str]) -> None: self._comment = info["comment"] elif rule == "api": self._api = info["api"] - elif rule == "artefacts": - self._artefacts = info["artefacts"] elif rule == "script_dir": self._relative_script_dir = info["script_dir"] else: @@ -134,7 +124,6 @@ def get_yaml_dict(self) -> dict[str, list[str]]: yaml_dict = {"comment": self.comment, "api": self.api, - "artefacts": self.artefacts, "script_dir": self._relative_script_dir} for rule, file_list in self._rules: diff --git a/lfric_build/tests/psyclone_control_test.py b/lfric_build/tests/psyclone_control_test.py index e32bb336e..ca68960cc 100644 --- a/lfric_build/tests/psyclone_control_test.py +++ b/lfric_build/tests/psyclone_control_test.py @@ -29,7 +29,6 @@ def test_psyclone_info_properties(): assert info.name == "test_phase" assert info.comment == "" assert info.api == "" - assert info.artefacts == "" assert info.opt_path == Path() @@ -56,7 +55,6 @@ def test_psyclone_control_read_and_to_yaml(tmp_path): dsl: comment: "PSyclone DSL Phase" api: lfric - artefacts: x90 script_dir: psykal global.py: \\* file_specific: \\* @@ -88,7 +86,6 @@ def test_psyclone_control_read_and_to_yaml(tmp_path): info_dsl = pc.get_info("dsl") assert info_dsl.comment == "PSyclone DSL Phase" assert info_dsl.api == "lfric" - assert info_dsl.artefacts == "x90" assert info_dsl.opt_path == script_root / "psykal" # Read second file to test incremental overrides From da6643bbad174f28b3e54978e7aece370f8988ac Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 29 Jun 2026 14:00:16 +1000 Subject: [PATCH 84/95] 371 Updated fixtures to use generic flags from Fab. --- lfric_build/tests/lfric_base_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index bf656ecf6..6e329fce0 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -64,8 +64,9 @@ def stub_fortran_compiler_init() -> FortranCompiler: Provides a minimal Fortran compiler. """ compiler = FortranCompiler('some Fortran compiler', 'sfc', 'stub', - r'([\d.]+)', openmp_flag='-omp', - module_folder_flag='-mods') + r'([\d.]+)') + compiler["openmp"] = '-omp' + compiler["module-out-folder"] = '-mods' return compiler @@ -75,7 +76,8 @@ def stub_c_compiler_init() -> CCompiler: Provides a minimal C compiler. """ compiler = CCompiler("some C compiler", "scc", "stub", - version_regex=r"([\d.]+)", openmp_flag='-omp') + version_regex=r"([\d.]+)") + compiler["openmp"] = '-omp' return compiler From c8aed18a830c2fb93d3ec6fddf137b70dfcf01f9 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Jul 2026 11:25:17 +1000 Subject: [PATCH 85/95] #371 Moved changes from dependency pr here, since they will be required for lfric_apps. Minor code updates. --- lfric_build/psyclone_control.py | 2 +- lfric_build/site_specific/default/config.py | 21 +++++++++++++++++++++ lfric_build/tests/lfric_base_test.py | 12 ++++++------ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 51b2a2cab..893e87851 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -211,7 +211,7 @@ def get_script(self, fpath: Path) -> Path: opt_script = self.opt_path / rule if not opt_script.exists(): raise FileNotFoundError(f"Cannot find script " - f"'{opt_script}'.") + f"'{opt_script}' for '{fpath}'.") return opt_script return PsycloneInfo.RESULT_EXCLUDE diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 57daf31f2..24d191539 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -98,6 +98,27 @@ def handle_command_line_options(self, args: argparse.Namespace) -> None: # initialising compilers self._args = args + def update_repos(self, dep_info): + """ + This method is called by the main script to allow each site to + replace the URLs of repos with e.g. local mirrors. + """ + + # A simplified example to use mirrors could be (which would + # typically be implemented in a derived, site-specific class) + # root = Path("/root/of/mirrors") + # mirrors = {"git@github.com:MetOffice/casim.git": root / "casim", + # "git@github.com:MetOffice/jules.git": root / "jules", + # } + # for dependency in dep_info.get_repo_names(): + # repo_infos = dep_info.get_repo_info(dependency) + # for source_ref in repo_infos: + # if source_ref.source in mirrors: + # logger.info(f"Using mirror " + # f"'{mirrors[source_ref.source]}' for " + # f"'{source_ref.source}") + # source_ref.source = mirrors[source_ref.source] + def setup_cray(self, build_config: BuildConfig) -> None: ''' This method sets up the Cray compiler and linker flags. diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 6e329fce0..fae15326b 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -40,6 +40,12 @@ def get_valid_profiles(self) -> List[str]: """ return ["default-profile"] + def update_repos(self, dep_info): + """ + This method is called by the main script to allow each site to + replace the URLs of repos with e.g. local mirrors. + """ + def update_toolbox(self, build_config: BuildConfig) -> None: """ Dummy function where the tool box could be modified @@ -51,12 +57,6 @@ def handle_command_line_options(self, args: argparse.Namespace) -> None: """ self.args = args - def get_path_flags(self, _build_config: BuildConfig) -> List[str]: - """ - :returns: list of path-specific flags. - """ - return [] - @pytest.fixture(name="stub_fortran_compiler", scope='function') def stub_fortran_compiler_init() -> FortranCompiler: From 36903c3111d432b0cdbcc2489d1b366655f7d4b1 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Jul 2026 11:44:20 +1000 Subject: [PATCH 86/95] #371 Also copy the optimisations script across to be consistent. --- lfric_build/lfric_base.py | 7 +++++-- lfric_build/tests/lfric_base_test.py | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 958023185..921c53ed7 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -247,8 +247,7 @@ def grab_files_step(self) -> None: ''' This method overwrites the base class grab_files_step. It includes all the LFRic core directories that are commonly required for building - LFRic applications. It also grabs the psydata directory for profiling, - if required. + LFRic applications. It also grabs optimisation scripts. ''' dirs = ['infrastructure/source/', 'components/driver/source/', @@ -266,6 +265,10 @@ def grab_files_step(self) -> None: grab_folder(self.config, src=self.lfric_core_root / "etc", dst_label='psyclone_config') + # Copy the optimisation scripts into a separate directory + grab_folder(self.config, src=self.app_dir / 'optimisation', + dst_label='optimisation') + def find_source_files_step( self, path_filters: Optional[Iterable[Union[Exclude, Include]]] = None diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index fae15326b..2c3396da1 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -413,7 +413,11 @@ def test_grab_files_step(monkeypatch) -> None: # PSyclone config directory mock.call(lfric_base.config, src=mock_core/'etc', - dst_label='psyclone_config') + dst_label='psyclone_config'), + # PSyclone optimisation scripts + mock.call(lfric_base.config, + src=Path('optimisation'), + dst_label='optimisation') ] # Check both number of calls and call arguments From 572bfb750e7610a2f948336a8e4633a95c137de4 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 1 Jul 2026 12:19:19 +1000 Subject: [PATCH 87/95] #371 Renamed psyclone_info.yaml to psyclone_control.yaml --- lfric_build/lfric_base.py | 14 +++++++------- lfric_build/psyclone_control.py | 6 +++--- .../{psyclone_info.yaml => psyclone_control.yaml} | 0 lfric_build/tests/lfric_base_test.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) rename lfric_build/{psyclone_info.yaml => psyclone_control.yaml} (100%) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 921c53ed7..1fb095707 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -103,14 +103,14 @@ def __init__(self, name: str, # values like NO_SCRIPT back for the PSyclone tool). self._current_psyclone_info: Optional[PsycloneInfo] = None - if self.args.psyclone_info: - info_list = [Path(i) for i in self.args.psyclone_info] + if self.args.psyclone_control: + control_list = [Path(i) for i in self.args.psyclone_control] else: # This default rule implements the "file-specific if exists, # otherwise global.py" rule. - info_list = [Path(self.lfric_core_root / "lfric_build" / - "psyclone_info.yaml")] - for psy_info_file in info_list: + control_list = [Path(self.lfric_core_root / "lfric_build" / + "psyclone_control.yaml")] + for psy_info_file in control_list: logger.info(f"Reading PSyclone configuration file " f"'{psy_info_file}'.") self._psyclone_control.read(Path(psy_info_file)) @@ -149,9 +149,9 @@ def define_command_line_options( help="Disable compilation with XIOS.") parser.add_argument( - '--psyclone-info', action="append", + '--psyclone-control', action="append", help="PSyclone configuration files, controlling when to " - "run PSyclone.") + "run the various PSyclone phases.") # Precision related command line arguments # ---------------------------------------- diff --git a/lfric_build/psyclone_control.py b/lfric_build/psyclone_control.py index 893e87851..ce5710ff9 100755 --- a/lfric_build/psyclone_control.py +++ b/lfric_build/psyclone_control.py @@ -219,9 +219,9 @@ def get_script(self, fpath: Path) -> Path: class PsycloneControl: """ - This class stores the information from psyclone_info.yaml file(s). Several - files can be read, and latter information will extend the rules from - previous files, and replace the phases executed. + This class stores the information from psyclone_control.yaml file(s). + Several files can be read, and latter information will extend the rules + from previous files, and replace the phases executed. Details of each phase will be stored in PsycloneInfo instances. diff --git a/lfric_build/psyclone_info.yaml b/lfric_build/psyclone_control.yaml similarity index 100% rename from lfric_build/psyclone_info.yaml rename to lfric_build/psyclone_control.yaml diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 2c3396da1..f5ace420f 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -228,7 +228,7 @@ def create_frame_info(filename): ] monkeypatch.setattr('inspect.stack', lambda: mock_stack) monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) - psyclone_control = mock_base_dir / "psyclone_info.yaml" + psyclone_control = mock_base_dir / "psyclone_control.yaml" psyclone_control.write_text("phases:", encoding='utf-8') lfric_base = LFRicBase(name="test", app_dir=tmp_path / "app_dir") From db20fd97ba0b3da41d02c66ae062f0af1f62b8c2 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 3 Aug 2026 15:02:22 +1000 Subject: [PATCH 88/95] #371 Updated documentation in psyclone control file. --- lfric_build/psyclone_control.yaml | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/lfric_build/psyclone_control.yaml b/lfric_build/psyclone_control.yaml index 98e0702d6..4fce37377 100644 --- a/lfric_build/psyclone_control.yaml +++ b/lfric_build/psyclone_control.yaml @@ -4,15 +4,8 @@ # First we specify the phases for PSyclone. This allows # any apps to run transmute and DSL steps in any order just by -# changing (or supplying a different) psyclone-info file. -# In this example, we run a transmute step before and after the DSL step. -# This allows transforming source code before PSyclone runs its DSL -# processing (during which transformed source code might be inlined). -# The third step is done to support feedback from the DSL processing -# to trigger additional processing (for example, if the DSL processing -# adds OpenACC directive, it might determine that additional source -# files need to be marked up to be compiled for OpenACC, which can then -# be done in an additional transmute phase). +# changing (or supplying a different) PSyclone-info file. +# In this file, we only run a single DSL phase. phases: # Just a single phase: running PSyclone in DSL mode @@ -22,13 +15,9 @@ dsl: comment: "PSylone DSL Phase" api: lfric - # Run on all x90 files - artefacts: x90 - script_dir: psykal - # The first two directives reproduce the default PSyclone triggering used - # in LFRic: + # These two directives reproduce the current LFRic default PSyclone triggering: # Run optimisation/.../global.py on all x90 files (x90 because this is # the dsl section, as specified in the artefacts above) @@ -37,6 +26,6 @@ dsl: # This represents existing functionality: if there is a file-specific .py # file, use it. The specification of '*' does not trigger an error # if there is no file_specific script (while explicitly putting a name - # here as shown in the previous pre_dsl phase will trigger an error, to - # catch typos early) + # in the file list will trigger an error if the corresponding PSyclone + # script does not exist. This way, errors are caught early). file_specific: \* From 957538025bb0286da699efa581910d2f23483e29 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 3 Aug 2026 16:22:09 +1000 Subject: [PATCH 89/95] #371 Added public function to add to the Python search path. --- lfric_build/lfric_base.py | 11 +++++++++++ lfric_build/tests/lfric_base_test.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 1fb095707..cce23515e 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -129,6 +129,17 @@ def lfric_core_root(self) -> Path: ''' return self._lfric_core_root + def add_python_search_path(self, path: Union[str, Path]) -> None: + ''' + Adds the specified path to the list of python search paths, which + will be added before PSyclone is executed. This allows an application + to add additional search paths required for the application (e.g. + the location of transmute scripts). + + :param path: the search path to add. + ''' + self._add_python_paths.append(str(path)) + def define_command_line_options( self, parser: Optional[argparse.ArgumentParser] = None diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index f5ace420f..004c9b506 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -238,6 +238,20 @@ def create_frame_info(filename): assert lfric_base.app_dir == tmp_path / "app_dir" +def test_python_search_path(monkeypatch): + ''' + Test that new paths can be added to the Python search path. + ''' + monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + lfric_base = LFRicBase(name="test_name", app_dir=Path(".")) + tools_path = str(lfric_base.lfric_core_root / "infrastructure" / "build" / + "psyclone") + # pylint: disable=protected-access + assert lfric_base._add_python_paths == [tools_path] + lfric_base.add_python_search_path("/special_path") + assert lfric_base._add_python_paths == [tools_path, "/special_path"] + + def test_require_openmp(monkeypatch, caplog) -> None: ''' Tests that using `-no-openmp` will abort with correct From 2ee4e86766bb511cb02a78065d0bd730f7587e9e Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 5 Aug 2026 12:49:05 +1000 Subject: [PATCH 90/95] #371 Minor code cleanup in preparation for apps-specific settings. --- lfric_build/lfric_base.py | 34 ++++++++------------- lfric_build/site_specific/default/config.py | 12 +++++--- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index cce23515e..cd63469cc 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -40,30 +40,35 @@ class LFRicBase(FabBase): :param app_dir: the base directory of the application. :param root_symbol: the symbol (or list of symbols) of the main programs. Defaults to the parameter `name` if not specified. + :param site_specific_dir: the base directory for the site-specific + files. If not specified, it will default to "directory + of the calling script" / site_specific ''' # pylint: disable=too-many-instance-attributes def __init__(self, name: str, app_dir: Path, - root_symbol: Optional[Union[list[str], str]] = None + root_symbol: Optional[Union[list[str], str]] = None, + site_specific_dir: Optional[Path] = None ): self._app_dir = app_dir + this_file = Path(__file__) + # The root directory of the LFRic Core + self._lfric_core_root = this_file.parents[1] + # List of all precision preprocessor symbols and their default. # Used to add corresponding command line options, and then to define # the preprocessor definitions. Note that precision_other - # becomes RDEF. + # becomes RDEF. Must be defined before calling super().__init__ + # (since it is required when defining command line options). self._all_precisions = [("precision_other", "64"), ("R_SOLVER_PRECISION", "32"), ("R_TRAN_PRECISION", "64"), ("R_BL_PRECISION", "64")] - super().__init__(name) - - this_file = Path(__file__) - # The root directory of the LFRic Core - self._lfric_core_root = this_file.parents[1] + super().__init__(name, site_specific_dir=site_specific_dir) # If the user wants to overwrite the default root symbol (which # is `name`): @@ -199,21 +204,6 @@ def handle_command_line_options(self, "command line.") sys.exit(-1) - def setup_site_specific_location(self): - ''' - This method adds the required directories for site-specific - configurations to the Python search path. We want to add the - directory where this lfric_base class is located, and not the - directory in which the application script is (which is what - baf base would set up). - ''' - this_dir = Path(__file__).parent - # We need to add the 'site_specific' directory to the path, so - # each config can import from 'default' (instead of having to - # use 'site_specific.default', which would hard-code the name - # `site_specific` in more scripts). - sys.path.insert(0, str(this_dir / "site_specific")) - def define_preprocessor_flags_step(self) -> None: ''' This method overwrites the base class define_preprocessor_flags. diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 24d191539..ff9f2807f 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -15,11 +15,13 @@ from fab.api import AddFlags, BuildConfig, Category, ToolRepository -from default.setup_script_cray import setup_script_cray -from default.setup_script_gnu import setup_script_gnu -from default.setup_script_intel_classic import setup_script_intel_classic -from default.setup_script_intel_llvm import setup_script_intel_llvm -from default.setup_script_nvidia import setup_script_nvidia +from site_specific.default.setup_script_cray import setup_script_cray +from site_specific.default.setup_script_gnu import setup_script_gnu +from site_specific.default.setup_script_intel_classic import ( + setup_script_intel_classic) +from site_specific.default.setup_script_intel_llvm import ( + setup_script_intel_llvm) +from site_specific.default.setup_script_nvidia import setup_script_nvidia class Config: From e34ccb6c26213ff4633cc02b19671e789d5448bd Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 10 Aug 2026 23:22:18 +1000 Subject: [PATCH 91/95] 371 Started to remove compilation flags that were required for lfric_atm. --- lfric_build/lfric_base.py | 2 +- lfric_build/site_specific/default/config.py | 11 ++--------- lfric_build/site_specific/default/setup_script_gnu.py | 10 ++-------- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index cd63469cc..50682ce93 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -68,7 +68,7 @@ def __init__(self, name: str, ("R_TRAN_PRECISION", "64"), ("R_BL_PRECISION", "64")] - super().__init__(name, site_specific_dir=site_specific_dir) + super().__init__(name) # If the user wants to overwrite the default root symbol (which # is `name`): diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index ff9f2807f..1e1181aa0 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -13,7 +13,7 @@ import argparse from typing import List -from fab.api import AddFlags, BuildConfig, Category, ToolRepository +from fab.api import BuildConfig, Category, ToolRepository from site_specific.default.setup_script_cray import setup_script_cray from site_specific.default.setup_script_gnu import setup_script_gnu @@ -141,6 +141,7 @@ def setup_gnu(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' + print("SiteConfig default GNU") setup_script_gnu(build_config, self.args) def setup_intel_classic(self, build_config: BuildConfig) -> None: @@ -175,11 +176,3 @@ def setup_nvidia(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' setup_script_nvidia(build_config, self.args) - - def get_path_flags(self, build_config: BuildConfig) -> List[AddFlags]: - ''' - Returns the path-specific flags to be used. - TODO FAB #313: Ideally we have only one kind of flag, but as a quick - work around we provide this method. - ''' - return [] diff --git a/lfric_build/site_specific/default/setup_script_gnu.py b/lfric_build/site_specific/default/setup_script_gnu.py index da935bc8b..a6b703aae 100644 --- a/lfric_build/site_specific/default/setup_script_gnu.py +++ b/lfric_build/site_specific/default/setup_script_gnu.py @@ -49,20 +49,14 @@ def setup_script_gnu(build_config: BuildConfig, # The base flags # ============== - # TODO: It should use -Werror=conversion, but: - # Most lfric_atm dependencies contain code with implicit lossy - # conversions. # This should be restricted to only the files/directories # that need it, but this needs Fab updates. gfortran.add_flags( ['-ffree-line-length-none', '-Wall', '-g', - '-Werror=character-truncation', - '-Werror=unused-value', - '-Werror=tabs', + '-Werror=character-truncation', '-Werror=unused-value', + '-Werror=tabs', '-Werror=conversion', '-std=f2008', - '-fdefault-real-8', - '-fdefault-double-8', ], "base") From e818c1443c0cb5cba4ca88a52ef4a0ea7011b3ad Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 18 Aug 2026 23:52:09 +1000 Subject: [PATCH 92/95] #371 Removed debug print. --- lfric_build/site_specific/default/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index 1e1181aa0..f8d111284 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -141,7 +141,6 @@ def setup_gnu(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' - print("SiteConfig default GNU") setup_script_gnu(build_config, self.args) def setup_intel_classic(self, build_config: BuildConfig) -> None: From 9db6e1c0e0174bf9324ad1eae51ff6b42536f920 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 19 Aug 2026 13:47:08 +1000 Subject: [PATCH 93/95] #371 Remove unused parameter (due to change in the corresponding Fab implementation). --- lfric_build/lfric_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lfric_build/lfric_base.py b/lfric_build/lfric_base.py index 50682ce93..fcdde1983 100755 --- a/lfric_build/lfric_base.py +++ b/lfric_build/lfric_base.py @@ -40,16 +40,12 @@ class LFRicBase(FabBase): :param app_dir: the base directory of the application. :param root_symbol: the symbol (or list of symbols) of the main programs. Defaults to the parameter `name` if not specified. - :param site_specific_dir: the base directory for the site-specific - files. If not specified, it will default to "directory - of the calling script" / site_specific ''' # pylint: disable=too-many-instance-attributes def __init__(self, name: str, app_dir: Path, root_symbol: Optional[Union[list[str], str]] = None, - site_specific_dir: Optional[Path] = None ): self._app_dir = app_dir From 38fcf6440d892683f501609d69a86ea6f23dbf6b Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 13:24:16 +1000 Subject: [PATCH 94/95] #371 Use simpler profiling definition. --- lfric_build/site_specific/default/config.py | 30 ++++++--------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/lfric_build/site_specific/default/config.py b/lfric_build/site_specific/default/config.py index f8d111284..292e3966d 100644 --- a/lfric_build/site_specific/default/config.py +++ b/lfric_build/site_specific/default/config.py @@ -13,7 +13,7 @@ import argparse from typing import List -from fab.api import BuildConfig, Category, ToolRepository +from fab.api import BuildConfig, ProfileFlags from site_specific.default.setup_script_cray import setup_script_cray from site_specific.default.setup_script_gnu import setup_script_gnu @@ -60,27 +60,13 @@ def update_toolbox(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance ''' - # First create the default compiler profiles for all available - # compilers. While we have a tool box with exactly one compiler - # in it, compiler wrappers will require more than one compiler - # to be initialised - so we just initialise all of them (including - # the linker): - tr = ToolRepository() - for compiler in (tr[Category.C_COMPILER] + - tr[Category.FORTRAN_COMPILER] + - tr[Category.LINKER]): - # Define a base profile, which contains the common - # compilation flags. This 'base' is not accessible to - # the user, so it's not part of the profile list. Also, - # make it inherit from the default profile '', so that - # a user does not have to specify the 'base' profile. - # Note that we set this even if a compiler is not available. - # This is required in case that compilers are not in PATH, - # so e.g. mpif90-ifort works, but ifort cannot be found. - # We still need to be able to set and query flags for ifort. - compiler.define_profile("base", inherit_from="") - for profile in self.get_valid_profiles(): - compiler.define_profile(profile, inherit_from="base") + # First create the default compiler profiles. + # Define a base profile, which contains the common + # compilation flags. This 'base' is not accessible to + # the user, so it's not part of the profile list. + ProfileFlags.define_profile("base") + for profile in self.get_valid_profiles(): + ProfileFlags.define_profile(profile, inherit_from="base") self.setup_intel_classic(build_config) self.setup_intel_llvm(build_config) From 91e7947593c5057f092d3866e17e41a56e46ca22 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 13:55:14 +1000 Subject: [PATCH 95/95] #371 Split tests into smaller pieces. --- lfric_build/tests/lfric_base_test.py | 161 +++++++++++++++------------ 1 file changed, 87 insertions(+), 74 deletions(-) diff --git a/lfric_build/tests/lfric_base_test.py b/lfric_build/tests/lfric_base_test.py index 004c9b506..b23d3bd68 100644 --- a/lfric_build/tests/lfric_base_test.py +++ b/lfric_build/tests/lfric_base_test.py @@ -26,7 +26,6 @@ from lfric_base import LFRicBase - class MockSiteConfig: """ Creates a mock site config class. @@ -368,7 +367,6 @@ def test_setup_site_specific_location(monkeypatch) -> None: # Check paths added correctly base_dir = Path(inspect.getfile(LFRicBase)).parent assert str(base_dir) in sys.path - assert str(base_dir / "site_specific") in sys.path # Restore path sys.path = old_path @@ -582,91 +580,106 @@ def test_get_rose_meta(monkeypatch) -> None: lfric_base = LFRicBase(name="test", app_dir=Path(".")) assert lfric_base.get_rose_meta() is None +class TestFullSetup(): -def test_analyse_step(monkeypatch) -> None: - '''Tests analysis step configuration and execution''' + def setup(self, monkeypatch, argv: Optional[list[str]] = None) -> None: + """ + """ + self.mpatch = monkeypatch + if not argv: + argv = ["lfric_base.py"] + self.mpatch.setattr(sys, "argv", argv) - # Test case 1: No ignore_dependencies argument specified - monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + # Create and setup mocks + self.mock_analyse = mock.MagicMock() + self.mpatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', + self.mock_analyse) - # Create mocks - mock_analyse = mock.MagicMock() - mock_preprocess = mock.MagicMock() - mock_psyclone = mock.MagicMock() + self.mock_preprocess = mock.MagicMock() + self.mpatch.setattr('lfric_base.preprocess_x90', self.mock_preprocess) - # Setup mocks - monkeypatch.setattr('fab.fab_base.fab_base.FabBase.analyse_step', - mock_analyse) + self.mock_psyclone_step = mock.MagicMock() + self.mpatch.setattr('lfric_base.LFRicBase.psyclone_step', + self.mock_psyclone_step) - lfric_base = LFRicBase(name="test", app_dir=Path(".")) + # Set up monkeypatch for module level import + self.mock_psyclone = mock.MagicMock() + monkeypatch.setattr('lfric_base.psyclone', self.mock_psyclone) - # Mock instance methods - monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) - monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) - - # The PSyclone step will modify sys.path (to allow import of - # psyclone_tools by PSyclone scripts). Make sure sys.path is unchanged: - old_sys_path = sys.path[:] - # Call analyse_step (which calls PSyclone) - lfric_base.analyse_step() - assert sys.path == old_sys_path - - # Verify method calls - mock_preprocess.assert_called_once() - mock_psyclone.assert_called_once() - - # Verify analyse called with correct default ignore_dependencies - expected_ignore = ['netcdf', 'mpi', 'mpi_f08', 'yaxt', - 'xios', 'icontext', 'mod_wait'] - mock_analyse.assert_called_once_with( - ignore_dependencies=expected_ignore, - find_programs=False - ) + self.lfric_base = LFRicBase(name="test", app_dir=Path(".")) - # Test case 2: Custom ignore_dependencies arguments specified - custom_ignore = ['custom_dep1', 'custom_dep2'] - mock_analyse.reset_mock() - mock_preprocess.reset_mock() - mock_psyclone.reset_mock() + self.mock_psyclone_config = "/mock/psyclone.cfg" + # Patch instance methods. Return a copy to avoid that + # PSyclone modified these lists in the lambdas when it modifies the list + monkeypatch.setattr(self.lfric_base, 'get_psyclone_config', + lambda: self.mock_psyclone_config) - lfric_base = LFRicBase(name="test", app_dir=Path(".")) - monkeypatch.setattr(lfric_base, 'preprocess_x90_step', mock_preprocess) - monkeypatch.setattr(lfric_base, 'psyclone_step', mock_psyclone) - - # Call analyse_step - lfric_base.analyse_step(ignore_dependencies=custom_ignore) - - # Verify methods still called - mock_preprocess.assert_called_once() - mock_psyclone.assert_called_once() - - # Verify analyse called with custom_ignore added to ignore list - expected_ignore = ['custom_dep1', 'custom_dep2', 'netcdf', 'mpi', - 'mpi_f08', 'yaxt', 'xios', 'icontext', - 'mod_wait'] - mock_analyse.assert_called_once_with( - ignore_dependencies=expected_ignore, - find_programs=False - ) + def test_analyse_no_ignore(self, monkeypatch) -> None: + """ + Tests analysis step configuration and execution, + if not additional dependencies are specified. + """ -def test_preprocess_x90_step(monkeypatch) -> None: - ''' - Tests preprocessing of X90 files. - ''' - monkeypatch.setattr(sys, "argv", ["lfric_base.py"]) + self.setup(monkeypatch) + + # The PSyclone step will modify sys.path (to allow import of + # psyclone_tools by PSyclone scripts). Make sure sys.path is unchanged: + old_sys_path = sys.path[:] + # Call analyse_step (which calls PSyclone) + self.lfric_base.analyse_step() + assert sys.path == old_sys_path + + # Verify method calls + self.mock_preprocess.assert_called_once() + self.mock_psyclone_step.assert_called_once() + + # Verify analyse called with correct default ignore_dependencies + expected_ignore = ['netcdf', 'mpi', 'mpi_f08', 'yaxt', + 'xios', 'icontext', 'mod_wait'] + self.mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) - mock_preproc = mock.MagicMock() - monkeypatch.setattr('lfric_base.preprocess_x90', mock_preproc) + def test_analyse_ignore_dependency(self, monkeypatch) -> None: + """ + Tests analysis step configuration and execution when + additional dependencies are specified. + """ - lfric_base = LFRicBase(name="test", app_dir=Path(".")) - lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) - lfric_base.preprocess_x90_step() + self.setup(monkeypatch) - mock_preproc.assert_called_once_with( - lfric_base.config, - common_flags=["-flag1", "-flag2"] - ) + # Call analyse_step (which calls PSyclone) + custom_ignore = ['custom_dep1', 'custom_dep2'] + self.lfric_base.analyse_step(ignore_dependencies=custom_ignore) + + # Verify method calls + self.mock_preprocess.assert_called_once() + self.mock_psyclone_step.assert_called_once() + + # Verify analyse called with correct default ignore_dependencies + expected_ignore = ['custom_dep1', 'custom_dep2', 'netcdf', 'mpi', + 'mpi_f08', 'yaxt', 'xios', 'icontext', + 'mod_wait'] + self.mock_analyse.assert_called_once_with( + ignore_dependencies=expected_ignore, + find_programs=False + ) + + def test_preprocess_x90_step(self, monkeypatch) -> None: + ''' + Tests preprocessing of X90 files. + ''' + self.setup(monkeypatch) + + self.lfric_base.add_preprocessor_flags(["-flag1", "-flag2"]) + self.lfric_base.preprocess_x90_step() + + self.mock_preprocess.assert_called_once_with( + self.lfric_base.config, + common_flags=["-flag1", "-flag2"] + ) def test_psyclone_step(monkeypatch) -> None: