From 5c527022ad9ddda2327335d4fc3b20b40d3e841c Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 13 Aug 2026 15:05:11 +1000 Subject: [PATCH 1/6] #594 Added command line options to skip checkouts or stop after checkouts. --- source/fab/fab_base/fab_base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/fab/fab_base/fab_base.py b/source/fab/fab_base/fab_base.py index d8799350..b666339d 100755 --- a/source/fab/fab_base/fab_base.py +++ b/source/fab/fab_base/fab_base.py @@ -472,6 +472,18 @@ class which can provide its own instance (to easily allow for a '--host', '-host', default="cpu", type=str, help="Determine the OpenACC or OpenMP: either 'cpu' or 'gpu'.") + parser.add_argument( + '--checkout-only', action="store_true", default=False, + help=("Only do the checkout steps, not any actual build steps." + "This can be useful if checkout and compilation steps " + "need to run on different nodes.")) + parser.add_argument( + '--skip-checkout', action="store_true", default=False, + help=("Do not do any checkouts. This flag can be used if a " + "checkout was already done, to just do the compilation. " + "This is useful if checkout and compilation needs to be " + "done on different nodes.")) + parser.add_argument("--site", "-s", type=str, default="$SITE or 'default'", help="Name of the site to use.") @@ -791,6 +803,10 @@ def build(self) -> None: # need to use it anywhere. with self._config as _: self.grab_files_step() + if self.args.checkout_only: + self.logger.info("Aborting after checkout due to " + "'--checkout-only' flag.") + return self.find_source_files_step() # This is a Fab function, which the user won't need to be # able to overwrite. From 9175e1a3ad504e132c03aa111f3123f9b9482fd6 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 13 Aug 2026 17:51:38 +1000 Subject: [PATCH 2/6] #594 Added test for new option --checkout-only. --- tests/unit_tests/fab_base/test_fab_base.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit_tests/fab_base/test_fab_base.py b/tests/unit_tests/fab_base/test_fab_base.py index 160d8c93..69134e25 100644 --- a/tests/unit_tests/fab_base/test_fab_base.py +++ b/tests/unit_tests/fab_base/test_fab_base.py @@ -8,6 +8,7 @@ """ import argparse import inspect +import logging import os from pathlib import Path import sys @@ -375,6 +376,41 @@ def test_site_specific_inside_dir(monkeypatch) -> None: assert "site_specific" == sys.path[0] +def test_checkout_only(monkeypatch, caplog) -> None: + ''' + Tests that FabBase does not run any build steps if + the --checkout-only flag is provided. + ''' + + monkeypatch.setattr(sys, "argv", ["fab_base.py", "--checkout-only"]) + + fab_base = FabBase(name="test") + + # We need to patch a lot of Fab functions (to avoid dependencies + # on the runtime environment): + mocks = {} + for function_name in ["grab_folder", "find_source_files", + "preprocess_c", "preprocess_fortran", + "compile_fortran", "compile_c", "analyse"]: + patcher = mock.patch(f"fab.fab_base.fab_base.{function_name}") + mocks[function_name] = (patcher, patcher.start()) + + with caplog.at_level(logging.INFO): + fab_base.build() + assert ("Aborting after checkout due to '--checkout-only' flag." + in caplog.text) + + mocks["grab_folder"][0].stop() + mocks["grab_folder"][1].assert_called_once_with( + fab_base.config, src=".") + # Check that no other function (except grab_folder) is being called. + for function_name, func_patcher in mocks.items(): + if function_name == "grab_folder": + continue + func_patcher[0].stop() + func_patcher[1].assert_not_called() + + def test_build_binary(monkeypatch) -> None: ''' Tests an actual trivial build. We patch all fab functions called From 2d9f4106aeacf8e519965153da9fe1ecf8de05eb Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 14 Aug 2026 11:17:32 +1000 Subject: [PATCH 3/6] #594 Added documentation. --- Documentation/source/fab_base/index.rst | 11 ++++- .../source/fab_base/usage_patterns.rst | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Documentation/source/fab_base/index.rst b/Documentation/source/fab_base/index.rst index 345eee6a..4660970f 100644 --- a/Documentation/source/fab_base/index.rst +++ b/Documentation/source/fab_base/index.rst @@ -37,7 +37,8 @@ description of the all options: .. parsed-literal:: usage: fab_base.py [-h] [--suite SUITE] [--available-compilers] [--fc FC] [--cc CC] [--ld LD] [--fflags FFLAGS] [--cflags CFLAGS] [--ldflags LDFLAGS] [--nprocs NPROCS] - [--mpi] [--no-mpi] [--openmp] [--no-openmp] [--openacc] [--host HOST] [--site SITE] [--platform PLATFORM] + [--mpi] [--no-mpi] [--openmp] [--no-openmp] [--openacc] [--host HOST] [--checkout-only] [--skip-checkout] [--site SITE] [--platform PLATFORM] + [--fab-workspace FAB_WORKSPACE] [--profile PROFILE] A Fab-based build system. Note that if --suite is specified, this will change the default for compiler and linker @@ -69,9 +70,17 @@ description of the all options: --openacc, -openacc Enable OpenACC (default: True) --host HOST, -host HOST Determine the OpenACC or OpenMP: either 'cpu' or 'gpu'. (default: cpu) + --checkout-only Only do the checkout steps, not any actual build steps.This can be useful if checkout and compilation steps need to run on different nodes. (default: + False) + --skip-checkout Do not do any checkouts. This flag can be used if a checkout was already done, to just do the compilation. This is useful if checkout and compilation + needs to be done on different nodes. (default: False) --site SITE, -s SITE Name of the site to use. (default: $SITE or 'default') --platform PLATFORM, -p PLATFORM Name of the platform of the site to use. (default: $PLATFORM or 'default') + --fab-workspace FAB_WORKSPACE + Fab workspace, in which the build directory will be created. (default: None) + --profile PROFILE, -pro PROFILE + Sets the compiler profile, choose from '['full-debug', 'fast-debug', 'production', 'unit-tests']'. (default: full-debug) Some command line option have an environment variable as default diff --git a/Documentation/source/fab_base/usage_patterns.rst b/Documentation/source/fab_base/usage_patterns.rst index 09cd2323..37893e3b 100644 --- a/Documentation/source/fab_base/usage_patterns.rst +++ b/Documentation/source/fab_base/usage_patterns.rst @@ -225,3 +225,44 @@ For example: linker = tr.get_tool(Category.LINKER, "linker-gfortran") linker.add_post_lib_flags(["-static-libasan"], "memory-debug") + +Running checkout and building independently +------------------------------------------- +On many platforms, only a few dedicated nodes might have internet access, +while the majority of compute nodes cannot access the internet at all. +In order to support these platforms, it is important that the checkout +of an application (e.g. using git) can be done without building, and +similarly that building can be executed without a checkout (meaning the +checkout must have ran before). + +The FabBase class provides two command line options to support this: + +1. ``--checkout-only`` + If this command line option is specified, Fab will exit (successfully) + after ``grab_files_step``. If the user should be running additional + tasks that require internet access, these must therefore be part of + ``grab_files_step``. A user code might need to check for this flag + (using ``fab_application.args.checkout_only``). + +2. ``--skip-checkout`` + This command line parameter is intended to avoid running any checkouts + (git, svn, ...). The Fab base class itself does not trigger any + checkouts, and so this flag is not actually used internally. It is the + responsibility of the application to implement this behaviour. + Example code for this: + + .. code-block:: python + + for repo_info in repo_infos: + if self.args.skip_checkout: + logger.info(f"Skipping extraction of '{repo}' from " + f"'{repo_info.source}' ") + continue + + logger.info(f"Extracting '{repo}' from '{repo_info.source}' " + f" to 'science/{repo}', " + f"revisions {repo_info.ref}") + git_checkout(self.config, + repo_info.source, + dst_label=f'science/{repo}', + revision=repo_info.ref) From 96d27c2a7ab7555f309997e84391267e58a40747 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 14 Aug 2026 18:55:23 +1000 Subject: [PATCH 4/6] #594 Updated contributors. --- CONTRIBUTORS.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 04815c18..9476d6e8 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -10,10 +10,11 @@ To indicate your agreement, add your details to the the following table. If you are not submitting contributions on behalf of an organisation please use "n/a" for your affiliation. -| GitHub Username | Real Name | Affiliation | -|-----------------|-----------------|-------------| -| MatthewHambley | Matthew Hambley | Met Office | -| yaswant | Yaswant Pradhan | Met Office | +| GitHub Username | Real Name | Affiliation | +|-----------------|-----------------|----------------------------------| +| MatthewHambley | Matthew Hambley | Met Office | +| yaswant | Yaswant Pradhan | Met Office | +| hiker | Joerg Henrichs | Bureau of Meteorology, Australia | --- From abc9d9e2f21f92a79c8fbb3b2a886a25a97bd175 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 24 Aug 2026 11:43:21 +1000 Subject: [PATCH 5/6] #594 Prevent --skip-checkout and --checkout-only from being used at the same time. --- source/fab/fab_base/fab_base.py | 5 +++-- tests/unit_tests/fab_base/test_fab_base.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/fab/fab_base/fab_base.py b/source/fab/fab_base/fab_base.py index b666339d..e1944472 100755 --- a/source/fab/fab_base/fab_base.py +++ b/source/fab/fab_base/fab_base.py @@ -472,12 +472,13 @@ class which can provide its own instance (to easily allow for a '--host', '-host', default="cpu", type=str, help="Determine the OpenACC or OpenMP: either 'cpu' or 'gpu'.") - parser.add_argument( + checkout_group = parser.add_mutually_exclusive_group() + checkout_group.add_argument( '--checkout-only', action="store_true", default=False, help=("Only do the checkout steps, not any actual build steps." "This can be useful if checkout and compilation steps " "need to run on different nodes.")) - parser.add_argument( + checkout_group.add_argument( '--skip-checkout', action="store_true", default=False, help=("Do not do any checkouts. This flag can be used if a " "checkout was already done, to just do the compilation. " diff --git a/tests/unit_tests/fab_base/test_fab_base.py b/tests/unit_tests/fab_base/test_fab_base.py index 69134e25..8127189e 100644 --- a/tests/unit_tests/fab_base/test_fab_base.py +++ b/tests/unit_tests/fab_base/test_fab_base.py @@ -411,6 +411,20 @@ def test_checkout_only(monkeypatch, caplog) -> None: func_patcher[1].assert_not_called() +def test_exclusive_checkout_skip(monkeypatch, capsys) -> None: + ''' + Tests that the flags --checkout-only and --skip-checkout + are exclusive. + ''' + monkeypatch.setattr(sys, "argv", ["fab_base.py", "--checkout-only", + "--skip-checkout"]) + with pytest.raises(SystemExit): + _ = FabBase(name="test-exclusive") + _, err = capsys.readouterr() + assert ("error: argument --skip-checkout: not allowed with argument " + "--checkout-only\n" in err) + + def test_build_binary(monkeypatch) -> None: ''' Tests an actual trivial build. We patch all fab functions called From b79f05ad9914adb5a5f95e8925cd0cf58b836b7a Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 24 Aug 2026 11:50:35 +1000 Subject: [PATCH 6/6] #594 Fixed failing tests after changes in main. --- tests/unit_tests/fab_base/test_fab_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit_tests/fab_base/test_fab_base.py b/tests/unit_tests/fab_base/test_fab_base.py index b01a1a33..79cfa563 100644 --- a/tests/unit_tests/fab_base/test_fab_base.py +++ b/tests/unit_tests/fab_base/test_fab_base.py @@ -389,7 +389,7 @@ def test_checkout_only(monkeypatch, caplog) -> None: # We need to patch a lot of Fab functions (to avoid dependencies # on the runtime environment): mocks = {} - for function_name in ["grab_folder", "find_source_files", + for function_name in ["grab_files", "find_source_files", "preprocess_c", "preprocess_fortran", "compile_fortran", "compile_c", "analyse"]: patcher = mock.patch(f"fab.fab_base.fab_base.{function_name}") @@ -400,12 +400,12 @@ def test_checkout_only(monkeypatch, caplog) -> None: assert ("Aborting after checkout due to '--checkout-only' flag." in caplog.text) - mocks["grab_folder"][0].stop() - mocks["grab_folder"][1].assert_called_once_with( + mocks["grab_files"][0].stop() + mocks["grab_files"][1].assert_called_once_with( fab_base.config, src=".") # Check that no other function (except grab_folder) is being called. for function_name, func_patcher in mocks.items(): - if function_name == "grab_folder": + if function_name == "grab_files": continue func_patcher[0].stop() func_patcher[1].assert_not_called()