diff --git a/bin/ecbundle b/bin/ecbundle index 642e9c2..32558ad 100755 --- a/bin/ecbundle +++ b/bin/ecbundle @@ -37,6 +37,9 @@ elif [[ "create" == "$1"* ]]; then elif [[ "populate" == "$1"* ]]; then shift ${SCRIPT_DIR}/ecbundle-populate "$@" +elif [[ "merge" == "$1"* ]]; then + shift + ${SCRIPT_DIR}/ecbundle-merge "$@" else echo "ERROR: Expected 'build' or 'create' or 'populate' as first argument" usage diff --git a/bin/ecbundle-merge b/bin/ecbundle-merge new file mode 100755 index 0000000..6a2572a --- /dev/null +++ b/bin/ecbundle-merge @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +""" +Script to merge and update bundle files +""" + +import os +import sys +from argparse import SUPPRESS, ArgumentParser, RawTextHelpFormatter + +sys.path.insert(0, os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/..')) +from ecbundle import BundleMerger +from ecbundle.logging import DEBUG, colors, error, logger, success + + +def main(): + + # Parse arguments + parser = ArgumentParser(description=__doc__, + formatter_class=RawTextHelpFormatter) + + # -------------------------------------------------------------------------- + # Parse common subcommands + # -------------------------------------------------------------------------- + parser.add_argument('--no-colour', '--no-color', + help='Disable color output', + action='store_true') + + parser.add_argument('--verbose', '-v', + help='Verbose output', + action='store_true') + + parser.add_argument('--bundle', + help='Configuration of bundle', default="bundle.yml") + + parser.add_argument('--bundle-update', + help='Configuration of bundle update', default="bundle-update.yml") + + parser.add_argument('-o', + help='output file', default="merged-bundle.yml") + + # -------------------------------------------------------------------------- + + # Close parser and populate variable args + args = parser.parse_args() + + # Explicitly disable coloured logs + if args.no_colour: + colors.disable() + + # Log everything, including commands executed + if args.verbose: + logger.setLevel(DEBUG) + + + errcode = 0 + + if BundleMerger(**vars(args)).merge() != 0: + errcode = 1 # error + + if errcode == 1: + error("\n!!! Errors occured !!!") + + return errcode + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ecbundle/__init__.py b/ecbundle/__init__.py index 1e2dde7..998c03f 100644 --- a/ecbundle/__init__.py +++ b/ecbundle/__init__.py @@ -12,6 +12,7 @@ from ecbundle.download import * # noqa from ecbundle.git import * # noqa from ecbundle.logging import * # noqa +from ecbundle.merge import * # noqa from ecbundle.option import * # noqa from ecbundle.populate import * # noqa from ecbundle.project import * # noqa diff --git a/ecbundle/merge.py b/ecbundle/merge.py new file mode 100644 index 0000000..73c6f65 --- /dev/null +++ b/ecbundle/merge.py @@ -0,0 +1,115 @@ +# (C) Copyright 2020- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +import copy +import os + +from .bundle import Bundle +from .logging import error, header, success +from .util import fullpath + +__all__ = ["BundleMerger"] + + +class BundleMerger(object): + def __init__(self, **kwargs): + self.config = kwargs + + def get(self, key, default=None): + return self.config[key] if self.config[key] is not None else default + + def deep_merge(self, original, updates): + """Recursively merge `updates` into `original`. + + Rules: + - Dictionaries and are merged recursively. + - lists and scalar values are replaced entirely. + - Keys missing from `updates` remain unchanged. + """ + if isinstance(original, dict) and isinstance(updates, dict): + merged = copy.deepcopy(original) + for key, value in updates.items(): + if key in merged: + if isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = self.deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + + else: + merged[key] = copy.deepcopy(value) + + else: + return copy.deepcopy(updates) + + return merged + + def bundle(self, update=False): + arg = "bundle" + if update: + arg += "_update" + bundle_path = fullpath(self.get(arg, None)) + if bundle_path: + if os.path.isfile(bundle_path): + return Bundle(bundle_path, env=True) + if not os.path.isdir(bundle_path): + error(f"ERROR: --{arg} argument is not a valid bundle file path") + return None + + return None + + def merge(self): + bundle = self.bundle() + bundle_update = self.bundle(update=True) + if not (bundle and bundle_update): + return 1 + + success("\nMerging bundle ") + header(f" {bundle_update.file()} into {bundle.file()}") + + # merging projects + project_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in bundle.projects() + } + + updated_project_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in bundle_update.projects() + } + + updated_dict = self.deep_merge(project_dict, updated_project_dict) + + bundle.config["projects"] = [ + {key: value} for key, value in updated_dict.items() + ] + + # merging options + option_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in bundle.options() + } + + updated_option_dict = { + item.config["name"]: {k: v for k, v in item.config.items() if k != "name"} + for item in bundle_update.options() + } + + updated_dict = self.deep_merge(option_dict, updated_option_dict) + + bundle.config["options"] = [{key: value} for key, value in updated_dict.items()] + + # merge remaining keys + + for key in bundle_update.config.keys(): + if key not in ["projects", "options"]: + bundle.config[key] = bundle_update.get(key) + + with open(self.get("o", None), "w", encoding="utf-8") as f: + f.write(bundle.yaml()) + + return 0 diff --git a/ecbundle/project.py b/ecbundle/project.py index dfe8fe5..a55b28e 100644 --- a/ecbundle/project.py +++ b/ecbundle/project.py @@ -74,6 +74,9 @@ def require(self): else: return None + def get_dict(self): + return self.config + def optional(self): return self.get("optional", False)