563 add dependency info

Documentation/source/fab_base/processing.rst

@@ -235,7 +235,10 @@ in this step various Fab functions are used to get the source files:
 
 ``git_checkout``
     Fab's ``git_checkout`` checks out a git repository, and puts the files
-    into the working directory.
+    into the working directory. Note that there is a convenience class
+    called ``DependencyInfo`` provided in Fab, that will manage the
+    yaml file provided in many suites from the UK MetOffice - see
+    :ref:`dependencies_yaml_support` for details.
 
 ``svn_export``, ``svn_checkout``
     Fab provides these two interfaces to svn, and similar to

Documentation/source/fab_base/usage_patterns.rst

@@ -266,3 +266,71 @@ The FabBase class provides two command line options to support this:
                          repo_info.source,
                          dst_label=f'science/{repo}',
                          revision=repo_info.ref)
+
+.. _dependencies_yaml_support:
+
+Using a UK Met Office ``dependencies.yaml`` file
+-------------------------------------------------
+Many UK Met office repositories, for example LFRic and UM,
+provide a ``dependencies.yaml`` file to specify dependencies
+on other repositories. Here an example:
+
+.. code-block:: yaml
+
+    casim:
+        source: git@github.com:MetOffice/casim.git
+        ref: 2026.07.1
+
+    jules:
+        source: git@github.com:MetOffice/jules.git
+        ref: 2026.07.1
+
+    SimSys_Scripts:
+        - source: git@github.com:MetOffice/SimSys_Scripts.git
+          ref: cab3315147a3c7e8546dda559d3da0fccd702f29
+        - source: git@github.com:MetOffice/SimSys_Scripts-fork.git
+          ref: feature-branch
+    ...
+
+Fab provides the class ``DependencyInfo`` to manage this kind
+of yaml file. Note that a dependcency can have more than one sources.
+This is typically used to merge several branches together before
+building.
+
+Example usage, taken from LFRic:
+
+.. code-block:: python
+
+    from fab.api import DependencyInfo
+    ...
+
+    def grab_files_step(self) -> None:
+
+        yaml_file = Path("/some/path/to/dep.yaml")
+        dep_info = DependencyInfo(yaml_file)
+
+        # Loop over all dependency repositories:
+        for repo, repo_infos in self.dependency_info.items():
+
+            # Each repo could have more than one branch listed,
+            # so we might need to extract more than one branch:
+            for repo_info in repo_infos:
+                logger.info(f"Extracting '{repo}' from '{repo_info.source}' "
+                            f" to 'science/{repo}', "
+                            f"revisions {repo_info.ref}")
+                try:
+                    git_checkout(self.config,
+                                 repo_info.source,
+                                 dst_label=f'science/{repo}',
+                                 revision=repo_info.ref)
+                except RuntimeError as error:
+                    logger.error(f"Cannot checkout '{repo}' from "
+                                 f"'{repo_info.source}' revision "
+                                 f"'{repo_info.ref}': {error}. ")
+                    sys.exit(-1)
+
+Note that this simplified example would just run ``git checkout``
+on the same directory repeatedly, it needs more sophisticated
+code in order to merge various branches together.
+The `SimSys_Script repository <https://github.com/MetOffice/SimSys_Scripts>`_
+contains code for this.

pyproject.toml

@@ -39,7 +39,9 @@ dev = [
     'pytest-subprocess >= 1.5.3',
     'pyfakefs',
     'flake8 >= 5.0.4',
-    'mypy'
+    'mypy',
+    'pyyaml',
+    'types-PyYAML'
 ]
 
 [project.scripts]

source/fab/api.py

@@ -20,6 +20,7 @@
 from fab.steps.compile_c import compile_c
 from fab.steps.compile_fortran import compile_fortran
 from fab.steps.find_source_files import Exclude, find_source_files, Include
+from fab.steps.grab.dependency_info import DependencyInfo
 from fab.steps.grab.fcm import fcm_export
 from fab.steps.grab.files import grab_files
 from fab.steps.grab.folder import grab_folder
@@ -63,6 +64,7 @@
     "ContainFlags",
     "c_pragma_injector",
     "Cpp",
+    "DependencyInfo",
     "Exclude",
     "FabBase",
     "fcm_export",

source/fab/steps/grab/dependency_info.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+##############################################################################
+# (c) Crown copyright Met Office. All rights reserved.
+# For further details please refer to the file COPYRIGHT
+# which you should have received as part of this distribution
+##############################################################################
+
+'''
+This module contains a class that manages the dependencies specified in
+a dependencies.yaml file.
+'''
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterator, Optional, Union
+import yaml
+
+
+@dataclass
+class RepoInfo:
+    """
+    A simple class to maintain source and ref attributes of a repository.
+    It maintains a list of sources and references, the idea being that the
+    first one is checked out, then the following are merged into the checked
+    out version.
+
+    This data is maintained in source_ref, a list of SourceRef instances.
+
+    Each RepoInfo also maintains one path, the location in which the source
+    code is checked out to.
+    """
+
+    @dataclass
+    class SourceRef:
+        """
+        A simple data class that stores a source and ref, and allows
+        to access and update them individually. Source and ref can
+        be None if there is no information for a repository.
+        """
+        source: Optional[str]
+        ref: Optional[str]
+
+    # The URL and reference of all sources
+    source_ref: list[SourceRef] = field(default_factory=list)
+
+    # The location on the file system where the checked out files are stored.
+    # Provide a default, this will be filled in later by the build script
+    # with the absolute path of the checked out version.
+    path: Path = Path(".")
+
+    def append(self, source: str, ref: str) -> None:
+        """
+        Provide a simple append function, to add a source and reference.
+
+        :param source: the source to add.
+        :param ref: the reference to use in this source
+        """
+        self.source_ref.append(RepoInfo.SourceRef(source, ref))
+
+    def __iter__(self) -> Iterator:
+        """
+        This function allows to iterate over all sources/references
+        of this dependency.
+        """
+        for item in self.source_ref:
+            yield item
+
+
+# ============================================================================
+class DependencyInfo(dict):
+    '''
+    A simple dictionary-like class that stores the version information
+    from a yaml file::
+
+        casim:
+            source: git@github.com:MetOffice/casim.git
+            ref: 2025.12.1
+        ...
+
+    The information can be accessed as a dictionary, e.g.::
+
+        gr = DependencyInfo("$LFRIC_APPS_SRC/dependencies.yaml")
+        gr["casim"] --> {"source": "git@.../casim.git",
+                         "ref": "2025.12.1"}
+
+    A filter can be specified to restrict the repositories that
+    are being handled.
+
+    The constructor will check that each dependency has indeed
+    source and ref defined (note that for lfric_apps these are
+    defined, but empty, indicating to use the current directory).
+
+    If the requested section does not exist, a key error is raised.
+
+    :param filename: The path to the dependencies.yaml file.
+    '''
+
+    def __init__(self, filename: Optional[Union[str, Path]],
+                 only_repos: Optional[list[str]] = None) -> None:
+        super().__init__()
+
+        # If there are no dependencies, just return (this object will
+        # then represents no dependencies).
+        if not filename:
+            return
+
+        with open(filename, "r", encoding="utf8") as stream:
+            dependencies = yaml.safe_load(stream)
+
+        for repo, all_deps in dependencies.items():
+            if only_repos and repo not in only_repos:
+                continue
+            # A repo can either have a single definition, or a list
+            # Support both:
+            if not isinstance(all_deps, list):
+                all_deps = [all_deps]
+
+            self[repo] = RepoInfo()
+            for dep in all_deps:
+                if "source" not in dep:
+                    raise RuntimeError(f"'{filename} does not contain a "
+                                       f"'source' definition for repo "
+                                       f"'{repo}'.")
+                if "ref" not in dep:
+                    raise RuntimeError(f"'{filename} does not contain a "
+                                       f"'ref' definition for repo '{repo}'.")
+                self[repo].append(dep["source"], dep["ref"])
+
+    def get_repo_names(self) -> list[str]:
+        """
+        :returns: the list of all repositories stored in this object.
+        """
+        return list(self.keys())
+
+    def get_repo_info(self, repo: str) -> RepoInfo:
+        """
+        :returns: the list of repository infos for a given dependency.
+
+        :raises KeyError: if the repository is not defined.
+        """
+        return self[repo]

tests/unit_tests/steps/grab/dependency_info_test.py

@@ -0,0 +1,141 @@
+##############################################################################
+# (c) Crown copyright Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+##############################################################################
+
+"""
+This module tests dependency_info.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from fab.steps.grab.dependency_info import DependencyInfo, RepoInfo
+
+
+def test_repo_info_append_and_iterate() -> None:
+    """
+    Check RepoInfo stores appended source/ref pairs and iterates over them.
+    """
+    repo_info = RepoInfo()
+
+    repo_info.append("git@example.com:first.git", "first-ref")
+    repo_info.append("git@example.com:second.git", "second-ref")
+
+    assert repo_info.path == Path(".")
+    assert list(repo_info) == [
+        RepoInfo.SourceRef("git@example.com:first.git", "first-ref"),
+        RepoInfo.SourceRef("git@example.com:second.git", "second-ref"),
+    ]
+
+
+@pytest.mark.parametrize("filename", [None, ""])
+def test_dependency_info_empty_filename(filename) -> None:
+    """
+    Check that no filename creates an empty dependency set.
+    """
+    dependency_info = DependencyInfo(filename)
+
+    assert dependency_info == {}
+    assert dependency_info.get_repo_names() == []
+    with pytest.raises(KeyError):
+        dependency_info.get_repo_info("missing")
+
+
+def test_dependency_info_reads_single_and_multiple_dependencies(
+        tmp_path: Path) -> None:
+    """
+    Check that both single dependency definitions and lists are supported.
+    """
+    dependency_file = tmp_path / "dependencies.yaml"
+    dependency_file.write_text(
+        "lfric_core:\n"
+        "    source:\n"
+        "    ref:\n"
+        "SimSys_Scripts:\n"
+        "    - source: git@github.com:MetOffice/SimSys_Scripts.git\n"
+        "      ref: cab3315147a3c7e8546dda559d3da0fccd702f29\n"
+        "    - source: git@github.com:MetOffice/SimSys_Scripts-fork.git\n"
+        "      ref: feature-branch\n",
+        encoding="utf8"
+    )
+
+    dependency_info = DependencyInfo(dependency_file)
+
+    assert dependency_info.get_repo_names() == ["lfric_core", "SimSys_Scripts"]
+    assert list(dependency_info.get_repo_info("lfric_core")) == [
+        RepoInfo.SourceRef(None, None)
+    ]
+    assert list(dependency_info.get_repo_info("SimSys_Scripts")) == [
+        RepoInfo.SourceRef(
+            "git@github.com:MetOffice/SimSys_Scripts.git",
+            "cab3315147a3c7e8546dda559d3da0fccd702f29",
+        ),
+        RepoInfo.SourceRef(
+            "git@github.com:MetOffice/SimSys_Scripts-fork.git",
+            "feature-branch",
+        ),
+    ]
+
+
+def test_dependency_info_only(tmp_path: Path) -> None:
+    """
+    Check that specifying 'only' works as expected
+    """
+    dependency_file = tmp_path / "dependencies.yaml"
+    dependency_file.write_text(
+        "repo1:\n"
+        "    source: git@bgithub.com/repo1\n"
+        "    ref: 1\n"
+        "repo2:\n"
+        "    source: git@bgithub.com/repo2\n"
+        "    ref: 2\n"
+        "repo3:\n"
+        "    source: git@bgithub.com/repo3\n"
+        "    ref: 3\n",
+        encoding="utf8"
+    )
+
+    dependency_info = DependencyInfo(dependency_file)
+    assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"]
+
+    dependency_info = DependencyInfo(dependency_file, ["repo1"])
+    assert dependency_info.get_repo_names() == ["repo1"]
+
+    dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2"])
+    assert dependency_info.get_repo_names() == ["repo1", "repo2"]
+
+    dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2",
+                                                       "repo3"])
+    assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"]
+
+
+@pytest.mark.parametrize(
+    "yaml_text, expected_message",
+    [
+        (
+            "test_repo:\n"
+            "    ref: test-ref\n",
+            "does not contain a 'source' definition for repo 'test_repo'",
+        ),
+        (
+            "test_repo:\n"
+            "    source: git@example.com:test.git\n",
+            "does not contain a 'ref' definition for repo 'test_repo'",
+        ),
+    ],
+)
+def test_dependency_info_rejects_missing_required_keys(
+        tmp_path: Path,
+        yaml_text: str,
+        expected_message: str) -> None:
+    """
+    Check that malformed dependency entries raise a helpful RuntimeError.
+    """
+    dependency_file = tmp_path / "dependencies.yaml"
+    dependency_file.write_text(yaml_text, encoding="utf8")
+
+    with pytest.raises(RuntimeError, match=expected_message):
+        DependencyInfo(dependency_file)