diff --git a/Documentation/source/fab_base/processing.rst b/Documentation/source/fab_base/processing.rst index 54b0b923..ee85c998 100644 --- a/Documentation/source/fab_base/processing.rst +++ b/Documentation/source/fab_base/processing.rst @@ -235,7 +235,10 @@ in this step various Fab functions are used to get the source files: ``git_checkout`` Fab's ``git_checkout`` checks out a git repository, and puts the files - into the working directory. + into the working directory. Note that there is a convenience class + called ``DependencyInfo`` provided in Fab, that will manage the + yaml file provided in many suites from the UK MetOffice - see + :ref:`dependencies_yaml_support` for details. ``svn_export``, ``svn_checkout`` Fab provides these two interfaces to svn, and similar to diff --git a/Documentation/source/fab_base/usage_patterns.rst b/Documentation/source/fab_base/usage_patterns.rst index 37893e3b..11648c7f 100644 --- a/Documentation/source/fab_base/usage_patterns.rst +++ b/Documentation/source/fab_base/usage_patterns.rst @@ -266,3 +266,71 @@ The FabBase class provides two command line options to support this: repo_info.source, dst_label=f'science/{repo}', revision=repo_info.ref) + +.. _dependencies_yaml_support: + +Using a UK Met Office ``dependencies.yaml`` file +------------------------------------------------- +Many UK Met office repositories, for example LFRic and UM, +provide a ``dependencies.yaml`` file to specify dependencies +on other repositories. Here an example: + +.. code-block:: yaml + + casim: + source: git@github.com:MetOffice/casim.git + ref: 2026.07.1 + + jules: + source: git@github.com:MetOffice/jules.git + ref: 2026.07.1 + + SimSys_Scripts: + - source: git@github.com:MetOffice/SimSys_Scripts.git + ref: cab3315147a3c7e8546dda559d3da0fccd702f29 + - source: git@github.com:MetOffice/SimSys_Scripts-fork.git + ref: feature-branch + ... + +Fab provides the class ``DependencyInfo`` to manage this kind +of yaml file. Note that a dependcency can have more than one sources. +This is typically used to merge several branches together before +building. + +Example usage, taken from LFRic: + +.. code-block:: python + + from fab.api import DependencyInfo + ... + + def grab_files_step(self) -> None: + + yaml_file = Path("/some/path/to/dep.yaml") + dep_info = DependencyInfo(yaml_file) + + # Loop over all dependency repositories: + for repo, repo_infos in self.dependency_info.items(): + + # Each repo could have more than one branch listed, + # so we might need to extract more than one branch: + for repo_info in repo_infos: + logger.info(f"Extracting '{repo}' from '{repo_info.source}' " + f" to 'science/{repo}', " + f"revisions {repo_info.ref}") + try: + git_checkout(self.config, + repo_info.source, + dst_label=f'science/{repo}', + revision=repo_info.ref) + except RuntimeError as error: + logger.error(f"Cannot checkout '{repo}' from " + f"'{repo_info.source}' revision " + f"'{repo_info.ref}': {error}. ") + sys.exit(-1) + +Note that this simplified example would just run ``git checkout`` +on the same directory repeatedly, it needs more sophisticated +code in order to merge various branches together. +The `SimSys_Script repository `_ +contains code for this. diff --git a/pyproject.toml b/pyproject.toml index 6cdaa89d..91d0b447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,9 @@ dev = [ 'pytest-subprocess >= 1.5.3', 'pyfakefs', 'flake8 >= 5.0.4', - 'mypy' + 'mypy', + 'pyyaml', + 'types-PyYAML' ] [project.scripts] diff --git a/source/fab/api.py b/source/fab/api.py index e16cf681..29b58c3f 100644 --- a/source/fab/api.py +++ b/source/fab/api.py @@ -20,6 +20,7 @@ from fab.steps.compile_c import compile_c from fab.steps.compile_fortran import compile_fortran from fab.steps.find_source_files import Exclude, find_source_files, Include +from fab.steps.grab.dependency_info import DependencyInfo from fab.steps.grab.fcm import fcm_export from fab.steps.grab.files import grab_files from fab.steps.grab.folder import grab_folder @@ -63,6 +64,7 @@ "ContainFlags", "c_pragma_injector", "Cpp", + "DependencyInfo", "Exclude", "FabBase", "fcm_export", diff --git a/source/fab/steps/grab/dependency_info.py b/source/fab/steps/grab/dependency_info.py new file mode 100755 index 00000000..0ae9ad9d --- /dev/null +++ b/source/fab/steps/grab/dependency_info.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# For further details please refer to the file COPYRIGHT +# which you should have received as part of this distribution +############################################################################## + +''' +This module contains a class that manages the dependencies specified in +a dependencies.yaml file. +''' + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator, Optional, Union +import yaml + + +@dataclass +class RepoInfo: + """ + A simple class to maintain source and ref attributes of a repository. + It maintains a list of sources and references, the idea being that the + first one is checked out, then the following are merged into the checked + out version. + + This data is maintained in source_ref, a list of SourceRef instances. + + Each RepoInfo also maintains one path, the location in which the source + code is checked out to. + """ + + @dataclass + class SourceRef: + """ + A simple data class that stores a source and ref, and allows + to access and update them individually. Source and ref can + be None if there is no information for a repository. + """ + source: Optional[str] + ref: Optional[str] + + # The URL and reference of all sources + source_ref: list[SourceRef] = field(default_factory=list) + + # The location on the file system where the checked out files are stored. + # Provide a default, this will be filled in later by the build script + # with the absolute path of the checked out version. + path: Path = Path(".") + + def append(self, source: str, ref: str) -> None: + """ + Provide a simple append function, to add a source and reference. + + :param source: the source to add. + :param ref: the reference to use in this source + """ + self.source_ref.append(RepoInfo.SourceRef(source, ref)) + + def __iter__(self) -> Iterator: + """ + This function allows to iterate over all sources/references + of this dependency. + """ + for item in self.source_ref: + yield item + + +# ============================================================================ +class DependencyInfo(dict): + ''' + A simple dictionary-like class that stores the version information + from a yaml file:: + + casim: + source: git@github.com:MetOffice/casim.git + ref: 2025.12.1 + ... + + The information can be accessed as a dictionary, e.g.:: + + gr = DependencyInfo("$LFRIC_APPS_SRC/dependencies.yaml") + gr["casim"] --> {"source": "git@.../casim.git", + "ref": "2025.12.1"} + + A filter can be specified to restrict the repositories that + are being handled. + + The constructor will check that each dependency has indeed + source and ref defined (note that for lfric_apps these are + defined, but empty, indicating to use the current directory). + + If the requested section does not exist, a key error is raised. + + :param filename: The path to the dependencies.yaml file. + ''' + + def __init__(self, filename: Optional[Union[str, Path]], + only_repos: Optional[list[str]] = None) -> None: + super().__init__() + + # If there are no dependencies, just return (this object will + # then represents no dependencies). + if not filename: + return + + with open(filename, "r", encoding="utf8") as stream: + dependencies = yaml.safe_load(stream) + + for repo, all_deps in dependencies.items(): + if only_repos and repo not in only_repos: + continue + # A repo can either have a single definition, or a list + # Support both: + if not isinstance(all_deps, list): + all_deps = [all_deps] + + self[repo] = RepoInfo() + for dep in all_deps: + if "source" not in dep: + raise RuntimeError(f"'{filename} does not contain a " + f"'source' definition for repo " + f"'{repo}'.") + if "ref" not in dep: + raise RuntimeError(f"'{filename} does not contain a " + f"'ref' definition for repo '{repo}'.") + self[repo].append(dep["source"], dep["ref"]) + + def get_repo_names(self) -> list[str]: + """ + :returns: the list of all repositories stored in this object. + """ + return list(self.keys()) + + def get_repo_info(self, repo: str) -> RepoInfo: + """ + :returns: the list of repository infos for a given dependency. + + :raises KeyError: if the repository is not defined. + """ + return self[repo] diff --git a/tests/unit_tests/steps/grab/dependency_info_test.py b/tests/unit_tests/steps/grab/dependency_info_test.py new file mode 100644 index 00000000..eab9849b --- /dev/null +++ b/tests/unit_tests/steps/grab/dependency_info_test.py @@ -0,0 +1,141 @@ +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +""" +This module tests dependency_info. +""" + +from pathlib import Path + +import pytest + +from fab.steps.grab.dependency_info import DependencyInfo, RepoInfo + + +def test_repo_info_append_and_iterate() -> None: + """ + Check RepoInfo stores appended source/ref pairs and iterates over them. + """ + repo_info = RepoInfo() + + repo_info.append("git@example.com:first.git", "first-ref") + repo_info.append("git@example.com:second.git", "second-ref") + + assert repo_info.path == Path(".") + assert list(repo_info) == [ + RepoInfo.SourceRef("git@example.com:first.git", "first-ref"), + RepoInfo.SourceRef("git@example.com:second.git", "second-ref"), + ] + + +@pytest.mark.parametrize("filename", [None, ""]) +def test_dependency_info_empty_filename(filename) -> None: + """ + Check that no filename creates an empty dependency set. + """ + dependency_info = DependencyInfo(filename) + + assert dependency_info == {} + assert dependency_info.get_repo_names() == [] + with pytest.raises(KeyError): + dependency_info.get_repo_info("missing") + + +def test_dependency_info_reads_single_and_multiple_dependencies( + tmp_path: Path) -> None: + """ + Check that both single dependency definitions and lists are supported. + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text( + "lfric_core:\n" + " source:\n" + " ref:\n" + "SimSys_Scripts:\n" + " - source: git@github.com:MetOffice/SimSys_Scripts.git\n" + " ref: cab3315147a3c7e8546dda559d3da0fccd702f29\n" + " - source: git@github.com:MetOffice/SimSys_Scripts-fork.git\n" + " ref: feature-branch\n", + encoding="utf8" + ) + + dependency_info = DependencyInfo(dependency_file) + + assert dependency_info.get_repo_names() == ["lfric_core", "SimSys_Scripts"] + assert list(dependency_info.get_repo_info("lfric_core")) == [ + RepoInfo.SourceRef(None, None) + ] + assert list(dependency_info.get_repo_info("SimSys_Scripts")) == [ + RepoInfo.SourceRef( + "git@github.com:MetOffice/SimSys_Scripts.git", + "cab3315147a3c7e8546dda559d3da0fccd702f29", + ), + RepoInfo.SourceRef( + "git@github.com:MetOffice/SimSys_Scripts-fork.git", + "feature-branch", + ), + ] + + +def test_dependency_info_only(tmp_path: Path) -> None: + """ + Check that specifying 'only' works as expected + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text( + "repo1:\n" + " source: git@bgithub.com/repo1\n" + " ref: 1\n" + "repo2:\n" + " source: git@bgithub.com/repo2\n" + " ref: 2\n" + "repo3:\n" + " source: git@bgithub.com/repo3\n" + " ref: 3\n", + encoding="utf8" + ) + + dependency_info = DependencyInfo(dependency_file) + assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"] + + dependency_info = DependencyInfo(dependency_file, ["repo1"]) + assert dependency_info.get_repo_names() == ["repo1"] + + dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2"]) + assert dependency_info.get_repo_names() == ["repo1", "repo2"] + + dependency_info = DependencyInfo(dependency_file, ["repo1", "repo2", + "repo3"]) + assert dependency_info.get_repo_names() == ["repo1", "repo2", "repo3"] + + +@pytest.mark.parametrize( + "yaml_text, expected_message", + [ + ( + "test_repo:\n" + " ref: test-ref\n", + "does not contain a 'source' definition for repo 'test_repo'", + ), + ( + "test_repo:\n" + " source: git@example.com:test.git\n", + "does not contain a 'ref' definition for repo 'test_repo'", + ), + ], +) +def test_dependency_info_rejects_missing_required_keys( + tmp_path: Path, + yaml_text: str, + expected_message: str) -> None: + """ + Check that malformed dependency entries raise a helpful RuntimeError. + """ + dependency_file = tmp_path / "dependencies.yaml" + dependency_file.write_text(yaml_text, encoding="utf8") + + with pytest.raises(RuntimeError, match=expected_message): + DependencyInfo(dependency_file) diff --git a/tests/unit_tests/test_api.py b/tests/unit_tests/test_api.py index d214653b..e95cf857 100644 --- a/tests/unit_tests/test_api.py +++ b/tests/unit_tests/test_api.py @@ -33,6 +33,7 @@ def test_import_from_api() -> None: "compile_c", "compile_fortran", "c_pragma_injector", + "DependencyInfo", "Exclude", "fcm_export", "file_checksum",