From 12cc7a593434356a83eb37982c28ad513e36257c Mon Sep 17 00:00:00 2001 From: pardallio Date: Fri, 29 May 2026 17:03:00 +0000 Subject: [PATCH 1/4] adds bundle merge functionality --- bin/ecbundle | 3 ++ bin/ecbundle-merge | 75 ++++++++++++++++++++++++++ ecbundle/__init__.py | 1 + ecbundle/merge.py | 126 +++++++++++++++++++++++++++++++++++++++++++ ecbundle/project.py | 3 ++ 5 files changed, 208 insertions(+) create mode 100755 bin/ecbundle-merge create mode 100644 ecbundle/merge.py diff --git a/bin/ecbundle b/bin/ecbundle index 642e9c2..32558ad 100755 --- a/bin/ecbundle +++ b/bin/ecbundle @@ -37,6 +37,9 @@ elif [[ "create" == "$1"* ]]; then elif [[ "populate" == "$1"* ]]; then shift ${SCRIPT_DIR}/ecbundle-populate "$@" +elif [[ "merge" == "$1"* ]]; then + shift + ${SCRIPT_DIR}/ecbundle-merge "$@" else echo "ERROR: Expected 'build' or 'create' or 'populate' as first argument" usage diff --git a/bin/ecbundle-merge b/bin/ecbundle-merge new file mode 100755 index 0000000..6a2572a --- /dev/null +++ b/bin/ecbundle-merge @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +""" +Script to merge and update bundle files +""" + +import os +import sys +from argparse import SUPPRESS, ArgumentParser, RawTextHelpFormatter + +sys.path.insert(0, os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/..')) +from ecbundle import BundleMerger +from ecbundle.logging import DEBUG, colors, error, logger, success + + +def main(): + + # Parse arguments + parser = ArgumentParser(description=__doc__, + formatter_class=RawTextHelpFormatter) + + # -------------------------------------------------------------------------- + # Parse common subcommands + # -------------------------------------------------------------------------- + parser.add_argument('--no-colour', '--no-color', + help='Disable color output', + action='store_true') + + parser.add_argument('--verbose', '-v', + help='Verbose output', + action='store_true') + + parser.add_argument('--bundle', + help='Configuration of bundle', default="bundle.yml") + + parser.add_argument('--bundle-update', + help='Configuration of bundle update', default="bundle-update.yml") + + parser.add_argument('-o', + help='output file', default="merged-bundle.yml") + + # -------------------------------------------------------------------------- + + # Close parser and populate variable args + args = parser.parse_args() + + # Explicitly disable coloured logs + if args.no_colour: + colors.disable() + + # Log everything, including commands executed + if args.verbose: + logger.setLevel(DEBUG) + + + errcode = 0 + + if BundleMerger(**vars(args)).merge() != 0: + errcode = 1 # error + + if errcode == 1: + error("\n!!! Errors occured !!!") + + return errcode + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ecbundle/__init__.py b/ecbundle/__init__.py index 1e2dde7..998c03f 100644 --- a/ecbundle/__init__.py +++ b/ecbundle/__init__.py @@ -12,6 +12,7 @@ from ecbundle.download import * # noqa from ecbundle.git import * # noqa from ecbundle.logging import * # noqa +from ecbundle.merge import * # noqa from ecbundle.option import * # noqa from ecbundle.populate import * # noqa from ecbundle.project import * # noqa diff --git a/ecbundle/merge.py b/ecbundle/merge.py new file mode 100644 index 0000000..4adbacf --- /dev/null +++ b/ecbundle/merge.py @@ -0,0 +1,126 @@ +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +import os +import copy +from collections import OrderedDict + +from .bundle import Bundle +from .logging import error, header, success +from .util import fullpath, mkdir_p, symlink_force + +__all__ = ["BundleMerger"] + + +class BundleMerger(object): + def __init__(self, **kwargs): + self.config = kwargs + + def get(self, key, default=None): + return self.config[key] if self.config[key] is not None else default + + def deep_merge(self, original, updates): + """Recursively merge `updates` into `original`. + + Rules: + - Dictionaries and are merged recursively. + - lists and scalar values are replaced entirely. + - Keys missing from `updates` remain unchanged. + """ + if isinstance(original, dict) and isinstance(updates, dict): + merged = copy.deepcopy(original) + for key, value in updates.items(): + if key in merged: + if isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = self.deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + + else: + merged[key] = copy.deepcopy(value) + + else: + return copy.deepcopy(updates) + + return merged + + def bundle(self,update=False): + arg="bundle" + if update: + arg += "_update" + bundle_path = fullpath(self.get(arg, None)) + if bundle_path: + if os.path.isfile(bundle_path): + return Bundle(bundle_path, env=True) + if not os.path.isdir(bundle_path): + error( + f"ERROR: --{arg} argument is not a valid bundle file path" + ) + return None + + return None + + def merge(self): + bundle = self.bundle() + bundle_update = self.bundle(update = True) + if not (bundle and bundle_update) : + return 1 + + success("\nMerging bundle ") + header(f" {bundle_update.file()} into {bundle.file()}") + + # merging projects + project_dict = { + item.config["name"]: { + k: v for k, v in item.config.items() if k != "name" + } + for item in bundle.projects() + } + + updated_project_dict = { + item.config["name"]: { + k: v for k, v in item.config.items() if k != "name" + } + for item in bundle_update.projects() + } + + updated_dict = self.deep_merge(project_dict,updated_project_dict) + + bundle.config["projects"] = [{key:value} for key,value in updated_dict.items()] + + # merging options + option_dict = { + item.config["name"]: { + k: v for k, v in item.config.items() if k != "name" + } + for item in bundle.options() + } + + updated_option_dict = { + item.config["name"]: { + k: v for k, v in item.config.items() if k != "name" + } + for item in bundle_update.options() + } + + updated_dict = self.deep_merge(option_dict,updated_option_dict) + + bundle.config["options"] = [{key:value} for key,value in updated_dict.items()] + + # merge remaining keys + + for key in bundle_update.config.keys(): + if key not in ["projects","options"]: + bundle.config[key] = bundle_update.get(key) + + with open(self.get("o", None), "w", encoding="utf-8") as f: + f.write(bundle.yaml()) + + return 0 + + diff --git a/ecbundle/project.py b/ecbundle/project.py index dfe8fe5..a55b28e 100644 --- a/ecbundle/project.py +++ b/ecbundle/project.py @@ -74,6 +74,9 @@ def require(self): else: return None + def get_dict(self): + return self.config + def optional(self): return self.get("optional", False) From 357036ba1424097a6f318c6ee1000c00bea056e9 Mon Sep 17 00:00:00 2001 From: pardallio Date: Mon, 1 Jun 2026 13:18:09 +0000 Subject: [PATCH 2/4] linting --- ecbundle/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecbundle/merge.py b/ecbundle/merge.py index 4adbacf..afba190 100644 --- a/ecbundle/merge.py +++ b/ecbundle/merge.py @@ -6,8 +6,8 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. -import os import copy +import os from collections import OrderedDict from .bundle import Bundle From cc01e162847ab4925f7048bf2c615df5c0bb1f33 Mon Sep 17 00:00:00 2001 From: pardallio Date: Mon, 1 Jun 2026 13:37:48 +0000 Subject: [PATCH 3/4] linting --- ecbundle/merge.py | 54 +++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/ecbundle/merge.py b/ecbundle/merge.py index afba190..d23bbad 100644 --- a/ecbundle/merge.py +++ b/ecbundle/merge.py @@ -23,7 +23,7 @@ def __init__(self, **kwargs): def get(self, key, default=None): return self.config[key] if self.config[key] is not None else default - + def deep_merge(self, original, updates): """Recursively merge `updates` into `original`. @@ -48,9 +48,9 @@ def deep_merge(self, original, updates): return copy.deepcopy(updates) return merged - - def bundle(self,update=False): - arg="bundle" + + def bundle(self, update=False): + arg = "bundle" if update: arg += "_update" bundle_path = fullpath(self.get(arg, None)) @@ -58,69 +58,59 @@ def bundle(self,update=False): if os.path.isfile(bundle_path): return Bundle(bundle_path, env=True) if not os.path.isdir(bundle_path): - error( - f"ERROR: --{arg} argument is not a valid bundle file path" - ) + error(f"ERROR: --{arg} argument is not a valid bundle file path") return None return None def merge(self): bundle = self.bundle() - bundle_update = self.bundle(update = True) - if not (bundle and bundle_update) : + bundle_update = self.bundle(update=True) + if not (bundle and bundle_update): return 1 success("\nMerging bundle ") header(f" {bundle_update.file()} into {bundle.file()}") - + # merging projects project_dict = { - item.config["name"]: { - k: v for k, v in item.config.items() if k != "name" - } + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} for item in bundle.projects() } - + updated_project_dict = { - item.config["name"]: { - k: v for k, v in item.config.items() if k != "name" - } + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} for item in bundle_update.projects() } - - updated_dict = self.deep_merge(project_dict,updated_project_dict) - bundle.config["projects"] = [{key:value} for key,value in updated_dict.items()] + updated_dict = self.deep_merge(project_dict, updated_project_dict) + + bundle.config["projects"] = [ + {key: value} for key, value in updated_dict.items() + ] # merging options option_dict = { - item.config["name"]: { - k: v for k, v in item.config.items() if k != "name" - } + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} for item in bundle.options() } updated_option_dict = { - item.config["name"]: { - k: v for k, v in item.config.items() if k != "name" - } + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} for item in bundle_update.options() } - - updated_dict = self.deep_merge(option_dict,updated_option_dict) - bundle.config["options"] = [{key:value} for key,value in updated_dict.items()] + updated_dict = self.deep_merge(option_dict, updated_option_dict) + + bundle.config["options"] = [{key: value} for key, value in updated_dict.items()] # merge remaining keys for key in bundle_update.config.keys(): - if key not in ["projects","options"]: + if key not in ["projects", "options"]: bundle.config[key] = bundle_update.get(key) with open(self.get("o", None), "w", encoding="utf-8") as f: f.write(bundle.yaml()) return 0 - - From 796ecb4b4cbe068bc35ac618a9630f918fada07a Mon Sep 17 00:00:00 2001 From: pardallio Date: Mon, 1 Jun 2026 13:40:50 +0000 Subject: [PATCH 4/4] linting --- ecbundle/merge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecbundle/merge.py b/ecbundle/merge.py index d23bbad..73c6f65 100644 --- a/ecbundle/merge.py +++ b/ecbundle/merge.py @@ -8,11 +8,10 @@ import copy import os -from collections import OrderedDict from .bundle import Bundle from .logging import error, header, success -from .util import fullpath, mkdir_p, symlink_force +from .util import fullpath __all__ = ["BundleMerger"]