From d12cc10a917c140727364f35d6e88cfc7090b2ad Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 13:19:30 +1000 Subject: [PATCH 1/7] #599 Define profiling modes in class variables, and move ProfileFlags into its own file. --- source/fab/api.py | 2 + source/fab/tools/flags.py | 193 ------------------ source/fab/tools/linker.py | 2 +- source/fab/tools/tool_with_flags.py | 3 +- tests/conftest.py | 13 +- .../fab_base/site_specific/default/config.py | 31 +-- tests/unit_tests/steps/test_compile_c.py | 2 +- tests/unit_tests/test_api.py | 1 + .../unit_tests/tools/test_compiler_wrapper.py | 4 +- tests/unit_tests/tools/test_flags.py | 178 +--------------- tests/unit_tests/tools/test_linker.py | 13 +- .../unit_tests/tools/test_tool_with_flags.py | 2 +- 12 files changed, 38 insertions(+), 406 deletions(-) diff --git a/source/fab/api.py b/source/fab/api.py index e16cf681..19bf0bb0 100644 --- a/source/fab/api.py +++ b/source/fab/api.py @@ -36,6 +36,7 @@ from fab.tools.linker import Linker from fab.tools.pkg_config import PkgConfig from fab.tools.preprocessor import Cpp, Fpp +from fab.tools.profile_flags import ProfileFlags from fab.tools.tool import Tool from fab.tools.tool_box import ToolBox from fab.tools.tool_repository import ToolRepository @@ -87,6 +88,7 @@ "preprocess_c", "preprocess_fortran", "preprocess_x90", + "ProfileFlags", "psyclone", "root_inc_files", "run_mp", diff --git a/source/fab/tools/flags.py b/source/fab/tools/flags.py index 0acf2f2d..2f15be3c 100644 --- a/source/fab/tools/flags.py +++ b/source/fab/tools/flags.py @@ -32,9 +32,6 @@ FlagList: Manages a list of flags, each of which is an instance of an AbstractFlag. -ProfileFlags: - Manages a set of flags for specific profiles, including inheritance. - Each tool uses a ProfileFlags instance. At runtime, the compilation steps will use the selected profile to get the Flags instance to use. The function `get_flags` will resolve the list of AbstractFlags by @@ -392,193 +389,3 @@ def remove_flag(self, remove_flag: str, has_parameter: bool = False): for flags in self: flags.remove_flag(remove_flag, has_parameter) - - -class ProfileFlags: - '''A list of flags that support a 'profile' to be used. If no profile is - specified, it will use "" (empty string) as 'profile'. All functions take - an optional profile parameter, so this class can also be used for tools - that do not need a profile. - - :param flags: optional flags to be added to this profile. - :param profile: optional profile to use if flags are specified, - defaults to "". - ''' - - def __init__(self: "ProfileFlags", - flags: Optional[Union[AbstractFlags, str, list[str]]] = None, - profile: str = "") -> None: - # Stores the flags for each profile mode. The key is the (lower case) - # name of the profile mode, and it contains a list of flags. - # Initialise the dict with the default (empty) profile - self._profiles: dict[str, FlagList] = {"": FlagList()} - - # This dictionary stores an optional inheritance, where one mode - # 'inherits' the flags from a different mode (recursively) - self._inherit_from: dict[str, str] = {} - - if flags: - if profile != "": - self.define_profile(profile) - self.add_flags(flags, profile) - - def get_flags(self, - config: Optional["BuildConfig"] = None, - file_path: Optional[Path] = None) -> list[str]: - ''' - This method returns the flags used for the specified file, - i.e. it will support path-specific flags. The BuildConfig - is added as parameter to get the profile, but also to - allow flags to use templated expressions `$relative` and - `$output` (the values are taken from the config object). - - :param config: the build config object. It stores the selected - compilation profile, and paths that can be used in templated - expressions. - :param file_path: path to the source file to compile. - ''' - if not file_path: - # If no path, provide a dummy path - file_path = Path() - if config: - profile = config.profile - else: - profile = "" - - all_flags = self[profile] - - resolved_flags = [] - for flags in all_flags: - resolved_flags.extend(flags.get_flags(config, file_path)) - - return resolved_flags - - def __getitem__(self, - profile: Optional[str] = None) -> list[AbstractFlags]: - '''Returns the flags for the requested profile. If profile is not - specified, the empty profile ("") will be used. It will also take - inheritance into account, so add flags (recursively) from inherited - profiles. But this function will not resolve the flags, i.e. replace - the AbstractFlags instances with a list of strings. - - :param profile: the optional profile to use. - - :raises KeyError: if a profile is specified it is not defined - ''' - if profile is None: - profile = "" - else: - profile = profile.lower() - - # First add any flags that we inherit. This will recursively call - # this __getitem__ to resolve inheritance chains. - if profile in self._inherit_from: - inherit_from = self._inherit_from[profile] - flags = self[inherit_from][:] - else: - flags = [] - # Now add the flags from this ProfileFlags. Note if no profile - # is specified, "" will be used as key, and this is always - # defined in the constructor of this object, so it will never - # raise an exception in this case - try: - flags.extend(self._profiles[profile]) - except KeyError as err: - raise KeyError(f"Profile '{profile}' is not defined.") from err - - return flags - - def define_profile(self, - name: str, - inherit_from: Optional[str] = None): - '''Defines a new profile name, and allows to specify if this new - profile inherit settings from an existing profile. If inherit_from - is specified, the newly defined profile will inherit from an existing - profile (including the default profile ""). - - :param name: Name of the profile to define. - :param inherit_from: Optional name of a profile to inherit - settings from. - ''' - if name in self._profiles: - raise KeyError(f"Profile '{name}' is already defined.") - self._profiles[name.lower()] = FlagList() - - if inherit_from is not None: - if inherit_from not in self._profiles: - raise KeyError(f"Inherited profile '{inherit_from}' is " - f"not defined.") - self._inherit_from[name.lower()] = inherit_from.lower() - - def add_flags(self, - new_flags: Union[AbstractFlags, str, list[str]], - profile: Optional[str] = None) -> None: - '''Adds the specified flags to the list of flags. - - :param new_flags: A single string or list of strings which are the - flags to be added. - ''' - if profile is None: - profile = "" - else: - profile = profile.lower() - - if profile not in self._profiles: - raise KeyError(f"add_flags: Profile '{profile}' is not defined.") - - if isinstance(new_flags, str): - new_flags = [new_flags] - - self._profiles[profile].add_flags(new_flags) - - def remove_flag(self, - remove_flag: str, - profile: Optional[str] = None, - has_parameter: bool = False): - '''Removes all occurrences of `remove_flag` in flags. - If `has_parameter` is defined, the next entry in flags will also be - removed, and if this object contains this flag+parameter without space - (e.g. `-J/tmp`), it will be correctly removed. Note that only the - flag itself must be specified, you cannot remove a flag only if a - specific parameter is given (i.e. `remove_flag="-J/tmp"` will not - work if this object contains `[...,"-J", "/tmp"]`). - - :param remove_flag: the flag to remove - :param has_parameter: if the flag to remove takes a parameter - ''' - - if not profile: - profile = "" - else: - profile = profile.lower() - - if profile not in self._profiles: - raise KeyError(f"remove_flag: Profile '{profile}' is not defined.") - - self._profiles[profile].remove_flag(remove_flag, has_parameter) - - def checksum(self, - config: Optional["BuildConfig"] = None, - file_path: Optional[Path] = None) -> int: - """ - :param config: the config object (used for templating) - :param file_path: the file path of the source file, used for - path-specific flags. - - :returns: a checksum of the flags. - """ - - if not file_path: - # If no path, provide a dummy path - file_path = Path() - if config: - profile = config.profile - else: - profile = "" - - if profile not in self._profiles: - raise KeyError(f"checksum: Profile '{profile}' is " - f"not defined.") - - resolve_flags: list[str] = self.get_flags(config, file_path) - return string_checksum(str(resolve_flags)) diff --git a/source/fab/tools/linker.py b/source/fab/tools/linker.py index 085dd5c0..6fdc95d0 100644 --- a/source/fab/tools/linker.py +++ b/source/fab/tools/linker.py @@ -16,7 +16,7 @@ from fab.build_config import BuildConfig from fab.tools.category import Category from fab.tools.compiler import Compiler -from fab.tools.flags import ProfileFlags +from fab.tools.profile_flags import ProfileFlags from fab.tools.compiler_suite_tool import CompilerSuiteTool diff --git a/source/fab/tools/tool_with_flags.py b/source/fab/tools/tool_with_flags.py index 43056940..58ca0e2e 100644 --- a/source/fab/tools/tool_with_flags.py +++ b/source/fab/tools/tool_with_flags.py @@ -13,7 +13,8 @@ from typing import Optional, TYPE_CHECKING, Union from fab.tools.category import Category -from fab.tools.flags import AbstractFlags, ProfileFlags +from fab.tools.flags import AbstractFlags +from fab.tools.profile_flags import ProfileFlags from fab.tools.tool import Tool if TYPE_CHECKING: diff --git a/tests/conftest.py b/tests/conftest.py index f591fd7e..48cbfe77 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ from fab.tools.category import Category from fab.tools.compiler import CCompiler, FortranCompiler from fab.tools.linker import Linker +from fab.tools.profile_flags import ProfileFlags from fab.tools.tool_box import ToolBox from fab.tools.tool_repository import ToolRepository @@ -176,12 +177,22 @@ def reset_tool_repository(stub_fortran_compiler): A fixture that resets the ToolRepository singleton (and esp. will remove existing compiler instance which might have had a state change in a test). It is automatically - applies to each function, to ensure all tests will execute + applied to each function, to ensure all tests will execute in parallel as well. """ ToolRepository._singleton = None +@fixture(scope="function", autouse=True) +def reset_compilation_profiles(stub_fortran_compiler): + """ + A fixture that resets the class variable in ProfileFlags. It is + automatically applied to each function, to ensure all tests will + execute as expected in parallel as well. + """ + ProfileFlags._inherit_from = {"": ""} + + @fixture(scope='function') def stub_tool_repository(stub_fortran_compiler, stub_c_compiler, diff --git a/tests/unit_tests/fab_base/site_specific/default/config.py b/tests/unit_tests/fab_base/site_specific/default/config.py index 18686da4..4aeb291a 100644 --- a/tests/unit_tests/fab_base/site_specific/default/config.py +++ b/tests/unit_tests/fab_base/site_specific/default/config.py @@ -8,8 +8,7 @@ import argparse from fab.build_config import BuildConfig -from fab.tools.category import Category -from fab.tools.tool_repository import ToolRepository +from fab.tools.profile_flags import ProfileFlags class Config: @@ -52,27 +51,13 @@ def update_toolbox(self, build_config: BuildConfig) -> None: :param build_config: the Fab build configuration instance :type build_config: :py:class:`fab.BuildConfig` ''' - # First create the default compiler profiles for all available - # compilers. While we have a tool box with exactly one compiler - # in it, compiler wrappers will require more than one compiler - # to be initialised - so we just initialise all of them (including - # the linker): - tr = ToolRepository() - for compiler in (tr[Category.C_COMPILER] + - tr[Category.FORTRAN_COMPILER] + - tr[Category.LINKER]): - # Define a base profile, which contains the common - # compilation flags. This 'base' is not accessible to - # the user, so it's not part of the profile list. Also, - # make it inherit from the default profile '', so that - # a user does not have to specify the "base" profile. - # Note that we set this even if a compiler is not available. - # This is required in case that compilers are not in PATH, - # so e.g. mpif90-ifort works, but ifort cannot be found. - # We still need to be able to set and query flags for ifort. - compiler.define_profile("base", inherit_from="") - for profile in self.get_valid_profiles(): - compiler.define_profile(profile, inherit_from="base") + # First create the default compiler profiles. + # Define a base profile, which contains the common + # compilation flags. This 'base' is not accessible to + # the user, so it's not part of the profile list. + ProfileFlags.define_profile("base") + for profile in self.get_valid_profiles(): + ProfileFlags.define_profile(profile, inherit_from="base") def define_command_line_options(self, parser: argparse.ArgumentParser) -> None: diff --git a/tests/unit_tests/steps/test_compile_c.py b/tests/unit_tests/steps/test_compile_c.py index 9729bd00..837f469e 100644 --- a/tests/unit_tests/steps/test_compile_c.py +++ b/tests/unit_tests/steps/test_compile_c.py @@ -17,7 +17,7 @@ from fab.parse.c import AnalysedC from fab.steps.compile_c import _get_obj_combo_hash, _compile_file, compile_c from fab.tools.category import Category -from fab.tools.flags import ProfileFlags +from fab.tools.profile_flags import ProfileFlags from fab.tools.tool_box import ToolBox diff --git a/tests/unit_tests/test_api.py b/tests/unit_tests/test_api.py index d214653b..36802928 100644 --- a/tests/unit_tests/test_api.py +++ b/tests/unit_tests/test_api.py @@ -54,6 +54,7 @@ def test_import_from_api() -> None: "preprocess_c", "preprocess_fortran", "preprocess_x90", + "ProfileFlags", "psyclone", "root_inc_files", "run_mp", diff --git a/tests/unit_tests/tools/test_compiler_wrapper.py b/tests/unit_tests/tools/test_compiler_wrapper.py index 2efc1a8b..3e783065 100644 --- a/tests/unit_tests/tools/test_compiler_wrapper.py +++ b/tests/unit_tests/tools/test_compiler_wrapper.py @@ -17,6 +17,7 @@ from fab.tools.compiler_wrapper import (CompilerWrapper, CrayCcWrapper, CrayFtnWrapper, Mpicc, Mpif90) +from fab.tools.profile_flags import ProfileFlags from tests.conftest import ExtendedRecorder, call_list, not_found_callback @@ -315,9 +316,8 @@ def test_compiler_wrapper_flags_with_add_arg(stub_c_compiler: CCompiler, subproc_record: ExtendedRecorder): '''Tests that flags set in the base compiler will be accessed in the wrapper if also additional flags are specified.''' + ProfileFlags.define_profile('default', inherit_from='') mpicc = Mpicc(stub_c_compiler) - stub_c_compiler.define_profile('default', inherit_from="") - mpicc.define_profile('default', inherit_from="") # Due to inheritance, this will give "-a -b" for gcc stub_c_compiler.add_flags(['-a']) stub_c_compiler.add_flags(['-b'], 'default') diff --git a/tests/unit_tests/tools/test_flags.py b/tests/unit_tests/tools/test_flags.py index 87ddcf86..8d82b22f 100644 --- a/tests/unit_tests/tools/test_flags.py +++ b/tests/unit_tests/tools/test_flags.py @@ -11,8 +11,7 @@ import pytest from fab.build_config import AddFlags -from fab.tools.flags import (AlwaysFlags, ContainFlags, FlagList, MatchFlags, - ProfileFlags) +from fab.tools.flags import (AlwaysFlags, ContainFlags, FlagList, MatchFlags) from fab.util import string_checksum @@ -163,181 +162,6 @@ def test_flags_checksum(): assert flags.checksum() == string_checksum(str(list_of_flags)) -def test_profile_flags_with_profile(): - '''Tests adding flags.''' - pf = ProfileFlags() - pf.define_profile("base") - assert pf["base"] == [] - pf.add_flags("-base", "base") - - assert len(pf["base"]) == 1 - assert isinstance(pf["base"][0], AlwaysFlags) - assert pf["base"][0].get_flags() == ["-base"] - - pf.add_flags(["-base2", "-base3"], "base") - assert len(pf["base"]) == 2 - assert isinstance(pf["base"][0], AlwaysFlags) - assert isinstance(pf["base"][1], AlwaysFlags) - assert pf["base"][0].get_flags() == ["-base"] - assert pf["base"][1].get_flags() == ["-base2", "-base3"] - - # Check that we get an exception if we specify a profile - # that does not exist - with pytest.raises(KeyError) as err: - _ = pf["does_not_exist"] - assert "Profile 'does_not_exist' is not defined" in str(err.value) - - -def test_profile_flags_constructor_args(): - '''Tests various constructor argument combinations.''' - pf = ProfileFlags("-g") - assert len(pf[""]) == 1 - assert isinstance(pf[""][0], AlwaysFlags) - assert pf[""][0].get_flags() == ["-g"] - - pf = ProfileFlags("-g", profile="prof") - assert pf[""] == [] - assert len(pf["prof"]) == 1 - assert isinstance(pf["prof"][0], AlwaysFlags) - assert pf["prof"][0].get_flags() == ["-g"] - - -def test_profile_flags_without_profile(): - '''Tests adding flags.''' - pf = ProfileFlags() - assert pf[""] == [] - assert pf[None] == [] - pf.add_flags("-base") - assert len(pf[""]) == 1 - assert isinstance(pf[""][0], AlwaysFlags) - assert pf[""][0].get_flags() == ["-base"] - pf.add_flags(["-base2", "-base3"]) - assert len(pf[""]) == 2 - assert pf[""][0].get_flags() == ["-base"] - assert pf[""][1].get_flags() == ["-base2", "-base3"] - - # Check that we get an exception if we specify a profile - with pytest.raises(KeyError) as err: - _ = pf["does_not_exist"] - assert "Profile 'does_not_exist' is not defined" in str(err.value) - - # Check that we get an exception if we try to inherit from a profile - # that does not exist - with pytest.raises(KeyError) as err: - pf.define_profile("new_profile", "does_not_exist") - assert ("Inherited profile 'does_not_exist' is not defined." - in str(err.value)) - - # Test that inheriting from the default profile "" works - pf.define_profile("from_default", "") - assert pf._inherit_from["from_default"] == "" - - -def test_profile_flags_inheriting(stub_configuration): - '''Tests adding flags.''' - pf = ProfileFlags() - pf.define_profile("base") - assert pf["base"] == [] - # And there should not be any inherited profile defined: - assert "base" not in pf._inherit_from - - pf.add_flags("-base", "base") - stub_configuration.set_profile("base") - assert pf.get_flags(stub_configuration) == ["-base"] - - pf.define_profile("derived", "base") - stub_configuration.set_profile("derived") - assert pf.get_flags(stub_configuration) == ["-base"] - assert pf._inherit_from["derived"] == "base" - pf.add_flags("-derived", "derived") - assert pf.get_flags(stub_configuration) == ["-base", "-derived"] - - pf.define_profile("derived2", "derived") - stub_configuration.set_profile("derived2") - assert pf.get_flags(stub_configuration) == ["-base", "-derived"] - pf.add_flags("-derived2", "derived2") - assert pf.get_flags(stub_configuration) == ["-base", "-derived", - "-derived2"] - - -def test_profile_flags_removing(stub_configuration): - '''Tests adding flags.''' - pf = ProfileFlags() - pf.define_profile("base") - assert pf["base"] == [] - pf.add_flags(["-base1", "-base2"], "base") - warn_message = "Removing managed flag '-base1'." - with pytest.warns(UserWarning, match=warn_message): - pf.remove_flag("-base1", "base") - stub_configuration.set_profile("base") - assert pf.get_flags(stub_configuration, Path()) == ["-base2"] - - # Try removing a flag that's not there. This should not - # cause any issues. - pf.remove_flag("-does-not-exist") - assert pf.get_flags(stub_configuration, Path()) == ["-base2"] - - pf.add_flags(["-base1", "-base2"]) - warn_message = "Removing managed flag '-base1'." - with pytest.warns(UserWarning, match=warn_message): - pf.remove_flag("-base1") - stub_configuration.set_profile("") - assert pf.get_flags(stub_configuration) == ["-base2"] - - -def test_profile_flags_checksum(stub_configuration): - '''Tests computation of the checksum.''' - pf = ProfileFlags() - pf.define_profile("base") - list_of_flags = ['one', 'two', 'three', 'four'] - pf.add_flags(list_of_flags, "base") - stub_configuration._profile = "base" - assert (pf.checksum(stub_configuration, Path()) == - string_checksum(str(list_of_flags))) - - # These flags get added to the "" profile, NOT base: - list_of_flags_new = ["five", "six"] - pf.add_flags(list_of_flags_new) - stub_configuration.set_profile("") - assert (pf.checksum(stub_configuration, Path()) == - string_checksum(str(list_of_flags_new))) - - # Test handling when no config is provided: - assert (pf.checksum(file_path=Path()) == - string_checksum(str(list_of_flags_new))) - - # Test handling when no file_path is provided: - assert (pf.checksum(stub_configuration) == - string_checksum(str(list_of_flags_new))) - - -def test_profile_flags_errors_invalid_profile_name(stub_configuration): - '''Tests that given undefined profile names will raise - KeyError in call functions. - ''' - pf = ProfileFlags() - pf.define_profile("base") - with pytest.raises(KeyError) as err: - pf.define_profile("base") - assert "Profile 'base' is already defined." in str(err.value) - - with pytest.raises(KeyError) as err: - pf.add_flags(["-some-flag"], "does not exist") - assert ("add_flags: Profile 'does not exist' is not defined." - in str(err.value)) - - with pytest.raises(KeyError) as err: - pf.remove_flag("-some-flag", "does not exist") - assert ("remove_flag: Profile 'does not exist' is not defined." - in str(err.value)) - - stub_configuration._profile = "does_not_exist" - with pytest.raises(KeyError) as err: - pf.checksum(stub_configuration, Path("/some/path")) - assert ("checksum: Profile 'does_not_exist' is not defined." - in str(err.value)) - - def test_old_addflags(): """ Tests that old-style AddFlags are converted to MatchFlags. diff --git a/tests/unit_tests/tools/test_linker.py b/tests/unit_tests/tools/test_linker.py index b7fada31..0d5a9977 100644 --- a/tests/unit_tests/tools/test_linker.py +++ b/tests/unit_tests/tools/test_linker.py @@ -19,6 +19,7 @@ from fab.tools.compiler import CCompiler, FortranCompiler from fab.tools.compiler_wrapper import CompilerWrapper, Mpif90 from fab.tools.linker import Linker +from fab.tools.profile_flags import ProfileFlags def test_c_linker(stub_c_compiler: CCompiler, @@ -389,9 +390,9 @@ def test_linker_profile_flags_inheriting(stub_c_compiler, linker_wrapper = Linker(stub_c_compiler_wrapper, linker=linker) count = 0 + ProfileFlags.define_profile("base") + ProfileFlags.define_profile("derived", "base") for compiler in [stub_c_compiler, stub_c_compiler_wrapper]: - compiler.define_profile("base") - compiler.define_profile("derived", "base") compiler.add_flags(f"-f{count}", "base") compiler.add_flags(f"-f{count+1}", "derived") count += 2 @@ -417,13 +418,13 @@ def test_linker_profile_modes(stub_linker): stub_linker._post_lib_flags["base"] assert "Profile 'base' is not defined" in str(err.value) - stub_linker.define_profile("base") + # Defining a profile should also work for the pre and post + # lib flags: + ProfileFlags.define_profile("base") assert stub_linker._pre_lib_flags["base"] == [] - assert "base" not in stub_linker._pre_lib_flags._inherit_from assert stub_linker._post_lib_flags["base"] == [] - assert "base" not in stub_linker._post_lib_flags._inherit_from - stub_linker.define_profile("full-debug", "base") + ProfileFlags.define_profile("full-debug", "base") assert stub_linker._pre_lib_flags["full-debug"] == [] assert stub_linker._pre_lib_flags._inherit_from["full-debug"] == "base" assert stub_linker._post_lib_flags["full-debug"] == [] diff --git a/tests/unit_tests/tools/test_tool_with_flags.py b/tests/unit_tests/tools/test_tool_with_flags.py index 33575b80..9edbcb49 100644 --- a/tests/unit_tests/tools/test_tool_with_flags.py +++ b/tests/unit_tests/tools/test_tool_with_flags.py @@ -12,7 +12,7 @@ import pytest from fab.tools.category import Category -from fab.tools.flags import ProfileFlags +from fab.tools.profile_flags import ProfileFlags from fab.tools.tool_with_flags import ToolWithFlags Category.add("CATEGORY_FOR_UNIT_TESTS") From 9941f9c7b8d69a314cd2561063e6ae1ed3b8ca91 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 14:30:34 +1000 Subject: [PATCH 2/7] #599 Updated documentation. --- Documentation/source/fab_base/config.rst | 26 +++++++------------ .../source/fab_base/usage_patterns.rst | 2 +- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/Documentation/source/fab_base/config.rst b/Documentation/source/fab_base/config.rst index 71f75f36..1a35b6d6 100644 --- a/Documentation/source/fab_base/config.rst +++ b/Documentation/source/fab_base/config.rst @@ -123,26 +123,20 @@ for all compilers and linkers: .. code-block:: python + from fab.api import ProfileFlags + def update_toolbox(self, build_config: BuildConfig) -> None: - for compiler in (tr[Category.C_COMPILER] + - tr[Category.FORTRAN_COMPILER] + - tr[Category.LINKER]): - compiler.define_profile("base", inherit_from="") - for profile in self.get_valid_profiles(): - compiler.define_profile(profile, inherit_from="base") + ProfileFlags.define_profile("base") + for profile in self.get_valid_profiles(): + ProfileFlags.define_profile(profile, inherit_from="base") This sets up a hierarchy where each of the valid compilation profiles -inherits from a ``base`` profile. And they are defined for all -compilers, even if they might not be available. This will make sure -that using compilation modes work in a Fab compiler wrapper, since -it is possible that the wrapped compiler is not available, i.e. -not in ``$PATH``, but the wrapper is. Additionally, using -``get_valid_profiles`` also means that any additional profiles defined -from a derived class will automatically be created. If a different -hierarchy is requested (e.g. ``memory-profile`` might want to inherit -from ``full-debug``, this needs to be updated in the inheriting -class). +inherits from a ``base`` profile. Using ``get_valid_profiles`` also means +that any additional profiles defined from a derived class will automatically +be created. If a different hierarchy is requested (e.g. ``memory-profile`` +might want to inherit from ``full-debug``, this needs to be updated in the +inheriting class). After the profiling modes, a ``default`` class should setup all compilers (including the various flags for the different diff --git a/Documentation/source/fab_base/usage_patterns.rst b/Documentation/source/fab_base/usage_patterns.rst index 37893e3b..6fd33b33 100644 --- a/Documentation/source/fab_base/usage_patterns.rst +++ b/Documentation/source/fab_base/usage_patterns.rst @@ -191,7 +191,7 @@ how a site can then add its own compilation profile: ''' Determines the list of all allowed compiler profiles. Here we add one additional profile `memory-debug`. Note that the default - setup will automatically create that mode for any available compiler. + setup will automatically create that profile mode for all tools. :returns List[str]: list of all supported compiler profiles. ''' From d3f5448f6887a9ee9bb80d27f5d6be92ec1c8cdb Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 14:37:37 +1000 Subject: [PATCH 3/7] #599 Updated error message to be more useful. --- source/fab/fab_base/fab_base.py | 4 +++- tests/unit_tests/fab_base/test_fab_base.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/fab/fab_base/fab_base.py b/source/fab/fab_base/fab_base.py index 911a2243..374d90d2 100755 --- a/source/fab/fab_base/fab_base.py +++ b/source/fab/fab_base/fab_base.py @@ -548,7 +548,9 @@ def handle_command_line_options(self, # profile in the site config file. if (self.args.profile and self.args.profile not in self._site_config.get_valid_profiles()): - raise RuntimeError(f"Invalid profile '{self.args.profile}") + valid = self._site_config.get_valid_profiles() + raise RuntimeError(f"Invalid profile '{self.args.profile}'. " + f"Valid profiles are: \"{valid}\".") if self.args.suite: tr.set_default_compiler_suite(self.args.suite) diff --git a/tests/unit_tests/fab_base/test_fab_base.py b/tests/unit_tests/fab_base/test_fab_base.py index 2637aa35..c85bccc8 100644 --- a/tests/unit_tests/fab_base/test_fab_base.py +++ b/tests/unit_tests/fab_base/test_fab_base.py @@ -176,7 +176,9 @@ def test_profile_invalid(monkeypatch) -> None: monkeypatch.setattr(sys, "argv", ["fab_base.py", "--profile", "invalid"]) with pytest.raises(RuntimeError) as err: _ = FabBase(name="test-help") - assert "Invalid profile 'invalid" == str(err.value) + assert ("Invalid profile 'invalid'. Valid profiles are: " + "\"['default-profile', 'full-debug', 'fast-debug', " + "'production']\"." == str(err.value)) def test_suite_no_compiler(monkeypatch) -> None: From f856c84c9b562c5a911484af350b9cd7bb430d50 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 14:46:51 +1000 Subject: [PATCH 4/7] #599 Fix unhandled warning in tests, removed left-over debug prints. --- tests/unit_tests/steps/test_compile_fortran.py | 16 +++++++--------- tests/unit_tests/tools/test_compiler.py | 1 - 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/unit_tests/steps/test_compile_fortran.py b/tests/unit_tests/steps/test_compile_fortran.py index f3f7e4ea..bb74682f 100644 --- a/tests/unit_tests/steps/test_compile_fortran.py +++ b/tests/unit_tests/steps/test_compile_fortran.py @@ -100,11 +100,13 @@ def test_vanilla(self, analysed_files, stub_tool_box: ToolBox, FlagList(), {}, syntax_only=True) - uncompiled_result = compile_pass(config=config, - compiled=compiled, - uncompiled=uncompiled, - mod_hashes=mod_hashes, - mp_common_args=mp_common_args) + with warns(UserWarning, + match="_metric_send_conn not set, cannot send metrics"): + uncompiled_result = compile_pass(config=config, + compiled=compiled, + uncompiled=uncompiled, + mod_hashes=mod_hashes, + mp_common_args=mp_common_args) assert Path('/fab/a.f90') not in compiled assert Path('/fab/b.f90') in compiled @@ -446,10 +448,6 @@ def test_deps_hash(self, content, fs: FakeFilesystem, fake_process: FakeProcess) expect_object_fpath = Path( '/fab/proj/build_output/_prebuild/foofile.106dc4756.o' ) - print("XX", res) - print("YY", analysed_file.fpath, expect_object_fpath) - # XX CompiledFile(foofile, /fab/proj/build_output/_prebuild/foofile.106dc4756.o) - # YY foofile /fab/proj/build_output/_prebuild/foofile.1ff6e93b3.o assert res == CompiledFile(input_fpath=analysed_file.fpath, output_fpath=expect_object_fpath) assert [call.args for call in record.calls] == [ diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index f6f7b8cb..40a39b00 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -124,7 +124,6 @@ def test_compiler_path_specific_flags(stub_configuration, fc = stub_fortran_compiler # Make sure we can get a version number for the stub compiler: fc._version = (1, 2) - print(fc.name, fc.get_version()) contain_flag = ContainFlags(pattern="myfile", flags=["-myflag"]) From 754116574d6eb2f580de26d9b9921a30886d147c Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 16:54:42 +1000 Subject: [PATCH 5/7] #599 Added missing test file. --- tests/unit_tests/tools/test_profile_flags.py | 209 +++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 tests/unit_tests/tools/test_profile_flags.py diff --git a/tests/unit_tests/tools/test_profile_flags.py b/tests/unit_tests/tools/test_profile_flags.py new file mode 100644 index 00000000..9737eb67 --- /dev/null +++ b/tests/unit_tests/tools/test_profile_flags.py @@ -0,0 +1,209 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +'''Tests the compiler implementation. +''' + +from pathlib import Path +import pytest + +from fab.tools.flags import AlwaysFlags +from fab.tools.profile_flags import ProfileFlags +from fab.util import string_checksum + + +def test_profile_flags_defining_profiles(): + """ + Tests defining profiles. + """ + assert ProfileFlags._inherit_from == {'': ''} + ProfileFlags.define_profile("base") + assert ProfileFlags._inherit_from == {'base': '', '': ''} + + ProfileFlags.define_profile("derived", inherit_from="base") + assert ProfileFlags._inherit_from == {'': '', + 'base': '', + 'derived': 'base'} + + # Creating an instance of ProfileFlags will add a profile + # inheriting from "". + ProfileFlags(profile="new_profile") + assert ProfileFlags._inherit_from == {'': '', + 'base': '', + 'derived': 'base', + 'new_profile': ''} + + pr1 = ProfileFlags(profile="another_profile", flags="-flag") + # This will have added a single AlwaysFlag instance, on which + # we call get_flags: + assert pr1["another_profile"][0].get_flags() == ["-flag"] + + # Trying to create an already existing profile should raise an error: + with pytest.raises(KeyError) as err: + ProfileFlags.define_profile('base') + assert "Profile 'base' is already defined" in str(err.value) + + # Inheriting from a non-existing profile should raise an error: + with pytest.raises(KeyError) as err: + ProfileFlags.define_profile("new", inherit_from="does_not_exist") + assert ("Inherited profile 'does_not_exist' is not defined" + in str(err.value)) + + # Check that we get an exception if we specify a profile + # that does not exist in __get_item__ + pf = ProfileFlags() + with pytest.raises(KeyError) as err: + _ = pf["does_not_exist"] + assert "Profile 'does_not_exist' is not defined" in str(err.value) + + +def test_profile_flags_inheritance(stub_configuration): + """ + Tests adding flags. + """ + ProfileFlags.define_profile("base") + ProfileFlags.define_profile("derived", inherit_from="base") + pf = ProfileFlags() + # First test that we can access all profiles in the instance: + assert pf[""] == [] + assert pf["base"] == [] + assert pf["derived"] == [] + + # Now add flags to the various profiles + pf.add_flags("-dummy") + stub_configuration.set_profile("derived") + assert pf.get_flags(stub_configuration) == ["-dummy"] + pf.add_flags("-base", profile="base") + assert pf.get_flags(stub_configuration) == ["-dummy", "-base"] + pf.add_flags("-derived", profile="derived") + assert pf.get_flags(stub_configuration) == ["-dummy", "-base", "-derived"] + + # And ensure that inherited profiles do not get the flags + # from derived profiles> + stub_configuration.set_profile("") + assert pf.get_flags(stub_configuration) == ["-dummy"] + stub_configuration.set_profile("base") + assert pf.get_flags(stub_configuration) == ["-dummy", "-base"] + + +def test_profile_flags_several_flags(): + """ + Test that adding several flags for the same profile will + add more AlwaysFlags instances + """ + + ProfileFlags.define_profile("base") + pf = ProfileFlags() + pf.add_flags("-base", profile="base") + + assert len(pf["base"]) == 1 + assert isinstance(pf["base"][0], AlwaysFlags) + assert pf["base"][0].get_flags() == ["-base"] + + pf.add_flags(["-base2", "-base3"], "base") + assert len(pf["base"]) == 2 + assert isinstance(pf["base"][0], AlwaysFlags) + assert isinstance(pf["base"][1], AlwaysFlags) + assert pf["base"][0].get_flags() == ["-base"] + assert pf["base"][1].get_flags() == ["-base2", "-base3"] + + +def test_profile_flags_without_profile(): + """ + Tests adding flags when using the default "" profile. + """ + pf = ProfileFlags() + assert pf[""] == [] + assert pf[None] == [] + pf.add_flags("-base") + assert len(pf[""]) == 1 + assert isinstance(pf[""][0], AlwaysFlags) + assert pf[""][0].get_flags() == ["-base"] + pf.add_flags(["-base2", "-base3"]) + assert len(pf[""]) == 2 + assert pf[""][0].get_flags() == ["-base"] + assert pf[""][1].get_flags() == ["-base2", "-base3"] + + +def test_profile_flags_removing(stub_configuration): + """ + Tests removing flags. + """ + pf = ProfileFlags() + pf.define_profile("base") + assert pf["base"] == [] + pf.add_flags(["-base1", "-base2"], "base") + warn_message = "Removing managed flag '-base1'." + with pytest.warns(UserWarning, match=warn_message): + pf.remove_flag("-base1", "base") + stub_configuration.set_profile("base") + assert pf.get_flags(stub_configuration, Path()) == ["-base2"] + + # Try removing a flag that's not there. This should not + # cause any issues. + pf.remove_flag("-does-not-exist") + assert pf.get_flags(stub_configuration, Path()) == ["-base2"] + + # Remove a single flag, even if it was added in the same add_flags + # call with other flags + pf.add_flags(["-base1", "-base2"]) + warn_message = "Removing managed flag '-base1'." + with pytest.warns(UserWarning, match=warn_message): + pf.remove_flag("-base1") + stub_configuration.set_profile("") + assert pf.get_flags(stub_configuration) == ["-base2"] + + # Trying to remove flag from a non-existing profile: + with pytest.raises(KeyError) as err: + pf.remove_flag("-some-flag", "does not exist") + assert ("remove_flag: Profile 'does not exist' is not defined." + in str(err.value)) + + +def test_profile_flags_checksum(stub_configuration): + '''Tests computation of the checksum.''' + pf = ProfileFlags() + pf.define_profile("base") + list_of_flags = ['one', 'two', 'three', 'four'] + pf.add_flags(list_of_flags, "base") + stub_configuration._profile = "base" + assert (pf.checksum(stub_configuration, Path()) == + string_checksum(str(list_of_flags))) + + # These flags get added to the "" profile, NOT base: + list_of_flags_new = ["five", "six"] + pf.add_flags(list_of_flags_new) + stub_configuration.set_profile("") + assert (pf.checksum(stub_configuration, Path()) == + string_checksum(str(list_of_flags_new))) + + # Test handling when no config is provided: + assert (pf.checksum(file_path=Path()) == + string_checksum(str(list_of_flags_new))) + + # Test handling when no file_path is provided: + assert (pf.checksum(stub_configuration) == + string_checksum(str(list_of_flags_new))) + + # Test checksum from a non-existing profile + stub_configuration._profile = "does_not_exist" + with pytest.raises(KeyError) as err: + pf.checksum(stub_configuration, Path("/some/path")) + assert ("checksum: Profile 'does_not_exist' is not defined." + in str(err.value)) + + +def test_profile_flags_errors_invalid_profile_name(): + '''Tests that given undefined profile names will raise + KeyError in call functions. + ''' + pf = ProfileFlags() + pf.define_profile("base") + + with pytest.raises(KeyError) as err: + pf.add_flags(["-some-flag"], "does not exist") + assert ("add_flags: Profile 'does not exist' is not defined." + in str(err.value)) From 22c86fa6058b30a908a3820f30f3dfa3706ef5b6 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 23:25:21 +1000 Subject: [PATCH 6/7] #599 Added missing source file. --- source/fab/tools/profile_flags.py | 235 ++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 source/fab/tools/profile_flags.py diff --git a/source/fab/tools/profile_flags.py b/source/fab/tools/profile_flags.py new file mode 100644 index 00000000..42d302ac --- /dev/null +++ b/source/fab/tools/profile_flags.py @@ -0,0 +1,235 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +''' +This file contains the ProfileFlag class used to manage command line flags +for tools, especially path-specific flags for compiler. A ProfileFlag +manages flags for specific profiles, including inheritance. + +Each tool uses a ProfileFlags instance. At runtime, the compilation steps +will use the selected profile to get the Flags inProfileFlags: + Manages a set of flags for specific profiles, including inheritance. +stance to use. +The function `get_flags` will resolve the list of AbstractFlags by +converting them from left to right into a list of strings. For example, +[AlwaysFlag("-g"), ContainFlags("-O3", pattern="special_file")] +will be convert to `["-g", "-O3"]` if the file contains the string +`special_file`, and otherwise it will be `["-g"]`. + +''' + +import logging +from pathlib import Path +from typing import Optional, Union + +from fab.tools.flags import AbstractFlags, FlagList +from fab.util import string_checksum + +from fab.build_config import BuildConfig + +logger = logging.getLogger(__name__) + + +class ProfileFlags: + '''A list of flags that support a 'profile' to be used. If no profile is + specified, it will use "" (empty string) as 'profile'. If a profile + is defined without an explicit inherit, this dummy profile "" will be + used (which implies that any flags specified with a profile will + eventually be used by any profile, so it's an implicit, common base class. + + All functions take an optional profile parameter, so this class can also + be used for tools that do not need a profile. + + :param flags: optional flags to be added to this profile. + :param profile: optional profile to use if flags are specified, + defaults to "". + ''' + + # This dictionary stores inheritance, where one mode + # 'inherits' the flags from a different mode (recursively). To + # avoid having to handle "" as special case, it is added here + # as an always available dummy profile. + _inherit_from: dict[str, str] = {"": ""} + + def __init__(self: "ProfileFlags", + flags: Optional[Union[AbstractFlags, str, list[str]]] = None, + profile: str = "") -> None: + # Stores the flags for each profile mode. The key is the (lower case) + # name of the profile mode, and it contains a list of flags. + # Initialise the dict with the default (empty) profile + self._profiles: dict[str, FlagList] = {"": FlagList()} + + if profile != "": + ProfileFlags.define_profile(profile) + if flags: + self.add_flags(flags, profile) + + @classmethod + def define_profile(cls, + name: str, + inherit_from: Optional[str] = None): + '''Defines a new profile name, and allows to specify if this new + profile inherit settings from an existing profile. If inherit_from + is specified, the newly defined profile will inherit from an existing + profile (including the default profile ""). + + :param name: Name of the profile to define. + :param inherit_from: Optional name of a profile to inherit + settings from. + ''' + if name in cls._inherit_from: + raise KeyError(f"Profile '{name}' is already defined.") + + if inherit_from is not None: + if inherit_from not in cls._inherit_from: + raise KeyError(f"Inherited profile '{inherit_from}' is " + f"not defined.") + cls._inherit_from[name.lower()] = inherit_from.lower() + else: + cls._inherit_from[name.lower()] = "" + + def get_flags(self, + config: Optional["BuildConfig"] = None, + file_path: Optional[Path] = None) -> list[str]: + ''' + This method returns the flags used for the specified file, + i.e. it will support path-specific flags. The BuildConfig + is added as parameter to get the profile, but also to + allow flags to use templated expressions `$relative` and + `$output` (the values are taken from the config object). + + :param config: the build config object. It stores the selected + compilation profile, and paths that can be used in templated + expressions. + :param file_path: path to the source file to compile. + ''' + if not file_path: + # If no path, provide a dummy path + file_path = Path() + if config: + profile = config.profile + else: + profile = "" + + all_flags = self[profile] + + resolved_flags = [] + for flags in all_flags: + resolved_flags.extend(flags.get_flags(config, file_path)) + + return resolved_flags + + def __getitem__(self, + profile: Optional[str] = None) -> list[AbstractFlags]: + '''Returns the flags for the requested profile. If profile is not + specified, the empty profile ("") will be used. It will also take + inheritance into account, so add flags (recursively) from inherited + profiles. But this function will not resolve the flags, i.e. replace + the AbstractFlags instances with a list of strings. + + :param profile: the optional profile to use. + + :raises KeyError: if a profile is specified it is not defined + ''' + if profile is None: + profile = "" + else: + profile = profile.lower() + + if profile and profile not in ProfileFlags._inherit_from: + raise KeyError(f"Profile '{profile}' is not defined") + + # First add any flags that we inherit. This will recursively call + # this __getitem__ to resolve inheritance chains. + + if profile: + inherit_from = self._inherit_from[profile] + flags = self[inherit_from][:] + else: + flags = [] + # Now add the flags from this ProfileFlags. Note if no profile + # is specified, "" will be used as key, and this is always + # defined in the constructor of this object, so it will never + # raise an exception in this case + if profile.lower() in self._profiles: + flags.extend(self._profiles[profile]) + return flags + + def add_flags(self, + new_flags: Union[AbstractFlags, str, list[str]], + profile: Optional[str] = None) -> None: + '''Adds the specified flags to the list of flags. + + :param new_flags: A single string or list of strings which are the + flags to be added. + ''' + if profile is None: + profile = "" + else: + profile = profile.lower() + + if profile and profile not in ProfileFlags._inherit_from: + raise KeyError(f"add_flags: Profile '{profile}' is not defined.") + + if profile not in self._profiles: + self._profiles[profile] = FlagList() + + if isinstance(new_flags, str): + new_flags = [new_flags] + + self._profiles[profile].add_flags(new_flags) + + def remove_flag(self, + remove_flag: str, + profile: Optional[str] = None, + has_parameter: bool = False): + '''Removes all occurrences of `remove_flag` in flags. + If `has_parameter` is defined, the next entry in flags will also be + removed, and if this object contains this flag+parameter without space + (e.g. `-J/tmp`), it will be correctly removed. Note that only the + flag itself must be specified, you cannot remove a flag only if a + specific parameter is given (i.e. `remove_flag="-J/tmp"` will not + work if this object contains `[...,"-J", "/tmp"]`). + + :param remove_flag: the flag to remove + :param has_parameter: if the flag to remove takes a parameter + ''' + + if not profile: + profile = "" + else: + profile = profile.lower() + + if profile not in self._profiles: + raise KeyError(f"remove_flag: Profile '{profile}' is not defined.") + + self._profiles[profile].remove_flag(remove_flag, has_parameter) + + def checksum(self, + config: Optional["BuildConfig"] = None, + file_path: Optional[Path] = None) -> int: + """ + :param config: the config object (used for templating) + :param file_path: the file path of the source file, used for + path-specific flags. + + :returns: a checksum of the flags. + """ + + if not file_path: + # If no path, provide a dummy path + file_path = Path() + if config: + profile = config.profile + else: + profile = "" + + if profile not in self._profiles: + raise KeyError(f"checksum: Profile '{profile}' is " + f"not defined.") + + resolve_flags: list[str] = self.get_flags(config, file_path) + return string_checksum(str(resolve_flags)) From e13a78852acd8c5dfd3f914b89cf2c3b90c510cd Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 31 Aug 2026 12:10:56 +1000 Subject: [PATCH 7/7] #599 Addressed issues raised in review. --- source/fab/tools/profile_flags.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/fab/tools/profile_flags.py b/source/fab/tools/profile_flags.py index 42d302ac..07c3a2f7 100644 --- a/source/fab/tools/profile_flags.py +++ b/source/fab/tools/profile_flags.py @@ -54,7 +54,7 @@ class ProfileFlags: # as an always available dummy profile. _inherit_from: dict[str, str] = {"": ""} - def __init__(self: "ProfileFlags", + def __init__(self, flags: Optional[Union[AbstractFlags, str, list[str]]] = None, profile: str = "") -> None: # Stores the flags for each profile mode. The key is the (lower case) @@ -80,16 +80,18 @@ def define_profile(cls, :param inherit_from: Optional name of a profile to inherit settings from. ''' + name = name.lower() if name in cls._inherit_from: raise KeyError(f"Profile '{name}' is already defined.") if inherit_from is not None: + inherit_from = inherit_from.lower() if inherit_from not in cls._inherit_from: raise KeyError(f"Inherited profile '{inherit_from}' is " f"not defined.") - cls._inherit_from[name.lower()] = inherit_from.lower() + cls._inherit_from[name] = inherit_from else: - cls._inherit_from[name.lower()] = "" + cls._inherit_from[name] = "" def get_flags(self, config: Optional["BuildConfig"] = None, @@ -106,9 +108,7 @@ def get_flags(self, expressions. :param file_path: path to the source file to compile. ''' - if not file_path: - # If no path, provide a dummy path - file_path = Path() + if config: profile = config.profile else: