From 648fbdfbe026a36cd0bad361a5401f4c9d561e1b Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 26 Aug 2026 10:35:41 +1000 Subject: [PATCH 1/7] #563 Added DependencyInfo class including tests. --- source/fab/api.py | 2 + source/fab/steps/grab/dependency_info.py | 140 +++++++++++++++++ .../steps/grab/dependency_info_test.py | 141 ++++++++++++++++++ tests/unit_tests/test_api.py | 1 + 4 files changed, 284 insertions(+) create mode 100755 source/fab/steps/grab/dependency_info.py create mode 100644 tests/unit_tests/steps/grab/dependency_info_test.py diff --git a/source/fab/api.py b/source/fab/api.py index e16cf681..29b58c3f 100644 --- a/source/fab/api.py +++ b/source/fab/api.py @@ -20,6 +20,7 @@ from fab.steps.compile_c import compile_c from fab.steps.compile_fortran import compile_fortran from fab.steps.find_source_files import Exclude, find_source_files, Include +from fab.steps.grab.dependency_info import DependencyInfo from fab.steps.grab.fcm import fcm_export from fab.steps.grab.files import grab_files from fab.steps.grab.folder import grab_folder @@ -63,6 +64,7 @@ "ContainFlags", "c_pragma_injector", "Cpp", + "DependencyInfo", "Exclude", "FabBase", "fcm_export", diff --git a/source/fab/steps/grab/dependency_info.py b/source/fab/steps/grab/dependency_info.py new file mode 100755 index 00000000..0e72daae --- /dev/null +++ b/source/fab/steps/grab/dependency_info.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +''' +This module contains a class that manages the dependencies specified in +a dependencies.yaml file. +''' + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator, Optional, Union +import yaml + + +@dataclass +class RepoInfo: + """ + A simple class to maintain source and ref attributes of a repository. + It maintains a list of sources and references, the idea being that the + first one is checked out, then the following are merged into the checked + out version. + + This data is maintained in source_ref, a list of SourceRef instances. + + Each RepoInfo also maintains one path, the location in which the source + code is checked out to. + """ + + @dataclass + class SourceRef: + """ + A simple data class that stores a source and ref, and allows + to access and update them individually. Source and ref can + are None if there is no information for a repository. + """ + source: Optional[str] + ref: Optional[str] + + # The URL and reference of all sources + source_ref: list[SourceRef] = field(default_factory=list) + + # The location on the file system where the checked out files are stored. + # Provide a default, this will be filled in later by the build script + # with the absolute path of the checked out version. + path: Path = Path(".") + + def append(self, source: str, ref: str) -> None: + """ + Provide a simple append function, to add a source and reference. + + :param source: the source to add. + :param ref: the reference to use in this source + """ + self.source_ref.append(RepoInfo.SourceRef(source, ref)) + + def __iter__(self) -> Iterator: + """ + This function allows to iterate over all sources/references + of this dependency. + """ + for item in self.source_ref: + yield item + + +# ============================================================================ +class DependencyInfo(dict): + ''' + A simple dictionary-like class that stores the version information + from a yaml file: + + casim: + source: git@github.com:MetOffice/casim.git + ref: 2025.12.1 + ... + + The information can be accessed as a dictionary, e.g.: + gr = DependencyInfo("$LFRIC_APPS_SRC/dependencies.yaml") + gr["casim"] --> {"source": "git@.../casim.git", + "ref": "2025.12.1"} + + A filter can be specified to restrict the repositories that + are being handled. + + The constructor will check that each dependency has indeed + source and ref defined (note that for lfric_apps these are + defined, but empty, indicating to use the current directory). + + If the requested section does not exist, a key error is raised. + + :param filename: The path to the dependencies.yaml file. + ''' + + def __init__(self, filename: Optional[Union[str, Path]], + only_repos: Optional[list[str]]) -> None: + super().__init__() + + # If there are no dependencies, just return (this object will + # then represents no dependencies). + if not filename: + return + + with open(filename, "r", encoding="utf8") as stream: + dependencies = yaml.safe_load(stream) + + for repo, all_deps in dependencies.items(): + if only_repos and repo not in only_repos: + continue + # A repo can either have a single definition, or a list + # Support both: + if not isinstance(all_deps, list): + all_deps = [all_deps] + + self[repo] = RepoInfo() + for dep in all_deps: + if "source" not in dep: + raise RuntimeError(f"'{filename} does not contain a " + f"'source' definition for repo " + f"'{repo}'.") + if "ref" not in dep: + raise RuntimeError(f"'{filename} does not contain a " + f"'ref' definition for repo '{repo}'.") + self[repo].append(dep["source"], dep["ref"]) + + def get_repo_names(self) -> list[str]: + """ + :returns: the list of all repositories stored in this object. + """ + return list(self.keys()) + + def get_repo_info(self, repo: str) -> RepoInfo: + """ + :returns: the list of repository infos for a given dependency. + + :raises:KeyError if the repository is not defined. + """ + return self[repo] diff --git a/tests/unit_tests/steps/grab/dependency_info_test.py b/tests/unit_tests/steps/grab/dependency_info_test.py new file mode 100644 index 00000000..6ddaf8c7 --- /dev/null +++ b/tests/unit_tests/steps/grab/dependency_info_test.py @@ -0,0 +1,141 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +""" +This module tests dependency_info. +""" + +from pathlib import Path + +import pytest + +from fab.steps.grab.dependency_info import DependencyInfo, RepoInfo + + +def test_repo_info_append_and_iterate() -> None: + """ + Check RepoInfo stores appended source/ref pairs and iterates over them. + """ + repo_info = RepoInfo() + + repo_info.append("git@example.com:first.git", "first-ref") + repo_info.append("git@example.com:second.git", "second-ref") + + assert repo_info.path == Path(".") + assert list(repo_info) == [ + RepoInfo.SourceRef("git@example.com:first.git", "first-ref"), + RepoInfo.SourceRef("git@example.com:second.git", "second-ref"), + ] + + +@pytest.mark.parametrize("filename", [None, ""]) +def test_dependency_info_empty_filename(filename) -> None: + """ + Check that no filename creates an empty dependency set. + """ + dependency_info = DependencyInfo(filename, []) + + assert dependency_info == {} + assert dependency_info.get_repo_names() == [] + with pytest.raises(KeyError): + dependency_info.get_repo_info("missing") + + +def test_dependency_info_reads_single_and_multiple_dependencies( + tmp_path: Path) -> None: + """ + Check that both single dependency definitions and lists are supported. + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text( + "lfric_core:\n" + " source:\n" + " ref:\n" + "SimSys_Scripts:\n" + " - source: git@github.com:MetOffice/SimSys_Scripts.git\n" + " ref: cab3315147a3c7e8546dda559d3da0fccd702f29\n" + " - source: git@github.com:MetOffice/SimSys_Scripts-fork.git\n" + " ref: feature-branch\n", + encoding="utf8" + ) + + dependency_info = DependencyInfo(dependency_file, []) + + assert dependency_info.get_repo_names() == ["lfric_core", "SimSys_Scripts"] + assert list(dependency_info.get_repo_info("lfric_core")) == [ + RepoInfo.SourceRef(None, None) + ] + assert list(dependency_info.get_repo_info("SimSys_Scripts")) == [ + RepoInfo.SourceRef( + "git@github.com:MetOffice/SimSys_Scripts.git", + "cab3315147a3c7e8546dda559d3da0fccd702f29", + ), + RepoInfo.SourceRef( + "git@github.com:MetOffice/SimSys_Scripts-fork.git", + "feature-branch", + ), + ] + + +def test_dependency_info_only(tmp_path: Path) -> None: + """ + Check that specifying 'only' works as expected + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text( + "repo1:\n" + " source: git@bgithub.com/repo1\n" + " ref: 1\n" + "repo2:\n" + " source: git@bgithub.com/repo2\n" + " ref: 2\n" + "repo3:\n" + " source: git@bgithub.com/repo3\n" + " ref: 3\n", + encoding="utf8" + ) + + dependency_info = DependencyInfo(dependency_file, []) + assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"] + + dependency_info = DependencyInfo(dependency_file, ["repo1"]) + assert dependency_info.get_repo_names() == ["repo1"] + + dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2"]) + assert dependency_info.get_repo_names() == ["repo1", "repo2"] + + dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2", + "repo3"]) + assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"] + + +@pytest.mark.parametrize( + "yaml_text, expected_message", + [ + ( + "test_repo:\n" + " ref: test-ref\n", + "does not contain a 'source' definition for repo 'test_repo'", + ), + ( + "test_repo:\n" + " source: git@example.com:test.git\n", + "does not contain a 'ref' definition for repo 'test_repo'", + ), + ], +) +def test_dependency_info_rejects_missing_required_keys( + tmp_path: Path, + yaml_text: str, + expected_message: str) -> None: + """ + Check that malformed dependency entries raise a helpful RuntimeError. + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text(yaml_text, encoding="utf8") + + with pytest.raises(RuntimeError, match=expected_message): + DependencyInfo(dependency_file, []) diff --git a/tests/unit_tests/test_api.py b/tests/unit_tests/test_api.py index d214653b..e95cf857 100644 --- a/tests/unit_tests/test_api.py +++ b/tests/unit_tests/test_api.py @@ -33,6 +33,7 @@ def test_import_from_api() -> None: "compile_c", "compile_fortran", "c_pragma_injector", + "DependencyInfo", "Exclude", "fcm_export", "file_checksum", From 8c37cafe5a95a62ff8ccc772ae73e8637ccc25e2 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 26 Aug 2026 15:21:08 +1000 Subject: [PATCH 2/7] #563 Updated documentation. --- Documentation/source/fab_base/processing.rst | 5 +- .../source/fab_base/usage_patterns.rst | 58 +++++++++++++++++++ source/fab/steps/grab/dependency_info.py | 7 ++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/Documentation/source/fab_base/processing.rst b/Documentation/source/fab_base/processing.rst index 54b0b923..ee85c998 100644 --- a/Documentation/source/fab_base/processing.rst +++ b/Documentation/source/fab_base/processing.rst @@ -235,7 +235,10 @@ in this step various Fab functions are used to get the source files: ``git_checkout`` Fab's ``git_checkout`` checks out a git repository, and puts the files - into the working directory. + into the working directory. Note that there is a convenience class + called ``DependencyInfo`` provided in Fab, that will manage the + yaml file provided in many suites from the UK MetOffice - see + :ref:`dependencies_yaml_support` for details. ``svn_export``, ``svn_checkout`` Fab provides these two interfaces to svn, and similar to diff --git a/Documentation/source/fab_base/usage_patterns.rst b/Documentation/source/fab_base/usage_patterns.rst index 37893e3b..366a0796 100644 --- a/Documentation/source/fab_base/usage_patterns.rst +++ b/Documentation/source/fab_base/usage_patterns.rst @@ -266,3 +266,61 @@ The FabBase class provides two command line options to support this: repo_info.source, dst_label=f'science/{repo}', revision=repo_info.ref) + +.. _dependencies_yaml_support: + +Using a UK Met Office ``dependencies.yaml`` file +------------------------------------------------- +Many UK Met office repositories, for example LFRic and UM, +provide a ``dependencies.yaml`` file to specify dependencies +on other repositories. Here a (shortened) example from +LFRic: + +.. code-block:: yaml + + casim: + source: git@github.com:MetOffice/casim.git + ref: 2026.07.1 + + jules: + source: git@github.com:MetOffice/jules.git + ref: 2026.07.1 + + lfric_core: + source: git@github.com:MetOffice/lfric_core.git + ref: 2026.07.1 + ... + +Fab provides the class ``DependencyInfo`` to manage this kind +of yaml file. Example usage, taken from LFRic: + +.. code-block:: python + + from fab.api import DependencyInfo + ... + + def grab_files_step(self) -> None: + + yaml_file = Path("/some/path/to/dep.yaml") + dep_info = DependencyInfo(yaml_file) + + # Loop over all dependency repositories: + for repo in self.dependency_info.get_repo_names(): + repo_infos = self.dependency_info.get_repo_info(repo) + + # Each repo could have more than one branch listed, + # so we might need to extract more than one branch: + for repo_info in repo_infos: + logger.info(f"Extracting '{repo}' from '{repo_info.source}' " + f" to 'science/{repo}', " + f"revisions {repo_info.ref}") + try: + git_checkout(self.config, + repo_info.source, + dst_label=f'science/{repo}', + revision=repo_info.ref) + except RuntimeError as error: + logger.error(f"Cannot checkout '{repo}' from " + f"'{repo_info.source}' revision " + f"'{repo_info.ref}': {error}. ") + sys.exit(-1) diff --git a/source/fab/steps/grab/dependency_info.py b/source/fab/steps/grab/dependency_info.py index 0e72daae..b1de314d 100755 --- a/source/fab/steps/grab/dependency_info.py +++ b/source/fab/steps/grab/dependency_info.py @@ -70,14 +70,15 @@ def __iter__(self) -> Iterator: class DependencyInfo(dict): ''' A simple dictionary-like class that stores the version information - from a yaml file: + from a yaml file:: casim: source: git@github.com:MetOffice/casim.git ref: 2025.12.1 ... - The information can be accessed as a dictionary, e.g.: + The information can be accessed as a dictionary, e.g.:: + gr = DependencyInfo("$LFRIC_APPS_SRC/dependencies.yaml") gr["casim"] --> {"source": "git@.../casim.git", "ref": "2025.12.1"} @@ -135,6 +136,6 @@ def get_repo_info(self, repo: str) -> RepoInfo: """ :returns: the list of repository infos for a given dependency. - :raises:KeyError if the repository is not defined. + :raises KeyError: if the repository is not defined. """ return self[repo] From 0d18f477a10dc06c7c7e5dc563f4a5fa923b9c48 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 26 Aug 2026 16:21:23 +1000 Subject: [PATCH 3/7] #563 Add pyyaml as dev dependency, --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6cdaa89d..c233d239 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ dev = [ 'pytest-subprocess >= 1.5.3', 'pyfakefs', 'flake8 >= 5.0.4', - 'mypy' + 'mypy', + 'pyyaml' ] [project.scripts] From c1235eaa722069399644151f3a89c9f369336251 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Wed, 26 Aug 2026 16:24:41 +1000 Subject: [PATCH 4/7] #563 Add types-pyyaml as dev dependency, --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c233d239..91d0b447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,8 @@ dev = [ 'pyfakefs', 'flake8 >= 5.0.4', 'mypy', - 'pyyaml' + 'pyyaml', + 'types-PyYAML' ] [project.scripts] From 35c474a0530466901ad0764d4c32f04796b11a6d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 27 Aug 2026 16:52:49 +1000 Subject: [PATCH 5/7] #563 Added missing default value, and corresponding tests. --- source/fab/steps/grab/dependency_info.py | 2 +- tests/unit_tests/steps/grab/dependency_info_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/fab/steps/grab/dependency_info.py b/source/fab/steps/grab/dependency_info.py index b1de314d..adedfd73 100755 --- a/source/fab/steps/grab/dependency_info.py +++ b/source/fab/steps/grab/dependency_info.py @@ -96,7 +96,7 @@ class DependencyInfo(dict): ''' def __init__(self, filename: Optional[Union[str, Path]], - only_repos: Optional[list[str]]) -> None: + only_repos: Optional[list[str]] = None) -> None: super().__init__() # If there are no dependencies, just return (this object will diff --git a/tests/unit_tests/steps/grab/dependency_info_test.py b/tests/unit_tests/steps/grab/dependency_info_test.py index 6ddaf8c7..eab9849b 100644 --- a/tests/unit_tests/steps/grab/dependency_info_test.py +++ b/tests/unit_tests/steps/grab/dependency_info_test.py @@ -36,7 +36,7 @@ def test_dependency_info_empty_filename(filename) -> None: """ Check that no filename creates an empty dependency set. """ - dependency_info = DependencyInfo(filename, []) + dependency_info = DependencyInfo(filename) assert dependency_info == {} assert dependency_info.get_repo_names() == [] @@ -62,7 +62,7 @@ def test_dependency_info_reads_single_and_multiple_dependencies( encoding="utf8" ) - dependency_info = DependencyInfo(dependency_file, []) + dependency_info = DependencyInfo(dependency_file) assert dependency_info.get_repo_names() == ["lfric_core", "SimSys_Scripts"] assert list(dependency_info.get_repo_info("lfric_core")) == [ @@ -98,7 +98,7 @@ def test_dependency_info_only(tmp_path: Path) -> None: encoding="utf8" ) - dependency_info = DependencyInfo(dependency_file, []) + dependency_info = DependencyInfo(dependency_file) assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"] dependency_info = DependencyInfo(dependency_file, ["repo1"]) @@ -138,4 +138,4 @@ def test_dependency_info_rejects_missing_required_keys( dependency_file.write_text(yaml_text, encoding="utf8") with pytest.raises(RuntimeError, match=expected_message): - DependencyInfo(dependency_file, []) + DependencyInfo(dependency_file) From 7b4b2b30a251fa71151d8725a0fb0814c6e70f72 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 31 Aug 2026 12:45:16 +1000 Subject: [PATCH 6/7] #563 Fixed typo. --- source/fab/steps/grab/dependency_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/fab/steps/grab/dependency_info.py b/source/fab/steps/grab/dependency_info.py index adedfd73..0ae9ad9d 100755 --- a/source/fab/steps/grab/dependency_info.py +++ b/source/fab/steps/grab/dependency_info.py @@ -35,7 +35,7 @@ class SourceRef: """ A simple data class that stores a source and ref, and allows to access and update them individually. Source and ref can - are None if there is no information for a repository. + be None if there is no information for a repository. """ source: Optional[str] ref: Optional[str] From c62686b88e5e5dd4961ab3dd4d78c24cdec66147 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 31 Aug 2026 13:05:51 +1000 Subject: [PATCH 7/7] #563 Updated documentation. --- .../source/fab_base/usage_patterns.rst | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/Documentation/source/fab_base/usage_patterns.rst b/Documentation/source/fab_base/usage_patterns.rst index 366a0796..11648c7f 100644 --- a/Documentation/source/fab_base/usage_patterns.rst +++ b/Documentation/source/fab_base/usage_patterns.rst @@ -273,8 +273,7 @@ Using a UK Met Office ``dependencies.yaml`` file ------------------------------------------------- Many UK Met office repositories, for example LFRic and UM, provide a ``dependencies.yaml`` file to specify dependencies -on other repositories. Here a (shortened) example from -LFRic: +on other repositories. Here an example: .. code-block:: yaml @@ -286,13 +285,19 @@ LFRic: source: git@github.com:MetOffice/jules.git ref: 2026.07.1 - lfric_core: - source: git@github.com:MetOffice/lfric_core.git - ref: 2026.07.1 + SimSys_Scripts: + - source: git@github.com:MetOffice/SimSys_Scripts.git + ref: cab3315147a3c7e8546dda559d3da0fccd702f29 + - source: git@github.com:MetOffice/SimSys_Scripts-fork.git + ref: feature-branch ... Fab provides the class ``DependencyInfo`` to manage this kind -of yaml file. Example usage, taken from LFRic: +of yaml file. Note that a dependcency can have more than one sources. +This is typically used to merge several branches together before +building. + +Example usage, taken from LFRic: .. code-block:: python @@ -305,8 +310,7 @@ of yaml file. Example usage, taken from LFRic: dep_info = DependencyInfo(yaml_file) # Loop over all dependency repositories: - for repo in self.dependency_info.get_repo_names(): - repo_infos = self.dependency_info.get_repo_info(repo) + for repo, repo_infos in self.dependency_info.items(): # Each repo could have more than one branch listed, # so we might need to extract more than one branch: @@ -324,3 +328,9 @@ of yaml file. Example usage, taken from LFRic: f"'{repo_info.source}' revision " f"'{repo_info.ref}': {error}. ") sys.exit(-1) + +Note that this simplified example would just run ``git checkout`` +on the same directory repeatedly, it needs more sophisticated +code in order to merge various branches together. +The `SimSys_Script repository `_ +contains code for this.