From 3f96ec36e6a0e0abf8db792193f9938e965a74d6 Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Thu, 24 Apr 2025 16:18:50 +0000 Subject: [PATCH 1/5] feat: Update UserMetadata - Add override method - Complete deep copy - Add dump --- src/earthkit/data/utils/metadata/dict.py | 53 ++++++++++++++++++- .../test_array_field_usermetadata.py | 44 +++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index fa22d020c..888536516 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -12,6 +12,7 @@ from math import prod import numpy as np +import copy from earthkit.data.core.geography import Geography from earthkit.data.core.metadata import Metadata @@ -452,7 +453,27 @@ def geography(self): return make_geography(self, values_shape=self._shape) def override(self, *args, **kwargs): - raise NotImplementedError("override is not implemented for UserMetadata") + r"""Create a new metadata object by cloning the underlying metadata and setting the keys in it. + + Parameters + ---------- + *args: tuple + Positional arguments. When present must be a dict with the keys to set in + the new metadata. + **kwargs: dict, optional + Other keyword arguments specifying the metadata keys to set. + + Returns + ------- + :class:`UserMetadata` + The new metadata object. A copy of the original metadata with the keys set in it. + """ + d = dict(*args, **kwargs) + d.copy() + existing = self._data.copy() + + existing.update(d) + return UserMetadata(existing, shape=copy.copy(self._shape)) def namespaces(self): return [] @@ -461,7 +482,35 @@ def as_namespace(self, namespace=None): return {} def dump(self, **kwargs): - return None + r"""Generate dump with all the metadata keys. + + In a Jupyter notebook it is represented as a tabbed interface. + + Parameters + ---------- + **kwargs: dict, optional + Other keyword arguments used for testing only + + Returns + ------- + NamespaceDump + Dict-like object with one item per namespace. In a Jupyter notebook represented + as a tabbed interface to browse the dump contents. + + Examples + -------- + :ref:`/examples/grib_metadata.ipynb` + + """ + from earthkit.data.utils.summary import format_namespace_dump + + r = [ + { + "title": "metadata", + "data": self._data, + } + ] + return format_namespace_dump(r, selected="parameter", details=self.__class__.__name__, **kwargs) def ls_keys(self): return self.LS_KEYS diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py index 2ac6146f8..bf1c5fd13 100644 --- a/tests/array_fieldlist/test_array_field_usermetadata.py +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -73,3 +73,47 @@ def test_array_field_usermetadata_geom(_kwargs): assert np.allclose(lat, meta["latitudes"]) assert np.allclose(lon, meta["longitudes"]) assert np.allclose(f.values, vals) + +@pytest.mark.parametrize( + "initial, update, expected", + [ + ({"shortName": "2t"}, {}, {"shortName": "2t"}), # No update + ({"shortName": "2t"}, {"shortName": "msl"}, {"shortName": "msl"}), # Update existing key + ({"shortName": "2t"}, {"longName": "Temperature"}, {"shortName": "2t", "longName": "Temperature"}), # Add new key + ({}, {"shortName": "2t"}, {"shortName": "2t"}), # Add key to empty metadata + ({"shortName": "2t", "longName": "Temperature"}, {"shortName": "temperature"}, {"shortName": "temperature", "longName": "Temperature"}), # Update one key, keep others + ], +) +def test_array_field_usermetadata_override(initial, update, expected): + meta = UserMetadata(initial) + new_meta = meta.override(update) + + # Check that the updated metadata matches the expected result + for k, v in expected.items(): + assert new_meta[k] == v + + # Ensure the original metadata remains unchanged + for k, v in initial.items(): + assert meta[k] == v + + # Check that the updated metadata contains all expected keys + for k in update: + if k in initial: + assert meta[k] == initial[k] + else: + assert k not in meta + +def test_array_field_usermetadata_override_shape(): + meta = UserMetadata( + {}, + shape = (10,1) + ) + new_meta = meta.override( + { + "shortName": "2t", + "longName": "Temperature", + } + ) + new_meta._shape = None + assert new_meta._shape is None + assert meta._shape is not None From 1319113e19f5c18578788523f5c517bb7f66d669 Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Thu, 24 Apr 2025 16:21:18 +0000 Subject: [PATCH 2/5] Use ** expansion in test --- tests/array_fieldlist/test_array_field_usermetadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py index bf1c5fd13..846e7c9c1 100644 --- a/tests/array_fieldlist/test_array_field_usermetadata.py +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -87,6 +87,7 @@ def test_array_field_usermetadata_geom(_kwargs): def test_array_field_usermetadata_override(initial, update, expected): meta = UserMetadata(initial) new_meta = meta.override(update) + _ = meta.override(**update) # Check that the updated metadata matches the expected result for k, v in expected.items(): From 5ae864b4909525083b9dd2b406e192390d17bfbf Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Thu, 24 Apr 2025 16:25:02 +0000 Subject: [PATCH 3/5] Run precommit --- src/earthkit/data/utils/metadata/dict.py | 2 +- .../test_array_field_usermetadata.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index 888536516..c4fddf5dd 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -7,12 +7,12 @@ # nor does it submit to any jurisdiction. # +import copy import logging from functools import cached_property from math import prod import numpy as np -import copy from earthkit.data.core.geography import Geography from earthkit.data.core.metadata import Metadata diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py index 846e7c9c1..be860a2b9 100644 --- a/tests/array_fieldlist/test_array_field_usermetadata.py +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -74,14 +74,23 @@ def test_array_field_usermetadata_geom(_kwargs): assert np.allclose(lon, meta["longitudes"]) assert np.allclose(f.values, vals) + @pytest.mark.parametrize( "initial, update, expected", [ ({"shortName": "2t"}, {}, {"shortName": "2t"}), # No update ({"shortName": "2t"}, {"shortName": "msl"}, {"shortName": "msl"}), # Update existing key - ({"shortName": "2t"}, {"longName": "Temperature"}, {"shortName": "2t", "longName": "Temperature"}), # Add new key + ( + {"shortName": "2t"}, + {"longName": "Temperature"}, + {"shortName": "2t", "longName": "Temperature"}, + ), # Add new key ({}, {"shortName": "2t"}, {"shortName": "2t"}), # Add key to empty metadata - ({"shortName": "2t", "longName": "Temperature"}, {"shortName": "temperature"}, {"shortName": "temperature", "longName": "Temperature"}), # Update one key, keep others + ( + {"shortName": "2t", "longName": "Temperature"}, + {"shortName": "temperature"}, + {"shortName": "temperature", "longName": "Temperature"}, + ), # Update one key, keep others ], ) def test_array_field_usermetadata_override(initial, update, expected): @@ -104,11 +113,9 @@ def test_array_field_usermetadata_override(initial, update, expected): else: assert k not in meta + def test_array_field_usermetadata_override_shape(): - meta = UserMetadata( - {}, - shape = (10,1) - ) + meta = UserMetadata({}, shape=(10, 1)) new_meta = meta.override( { "shortName": "2t", From f655d46f33556a2d5eaaecd430243029aa5db4a3 Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Wed, 30 Apr 2025 12:11:10 +0000 Subject: [PATCH 4/5] Address comments --- src/earthkit/data/utils/metadata/dict.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index c4fddf5dd..9cb65e388 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -469,11 +469,10 @@ def override(self, *args, **kwargs): The new metadata object. A copy of the original metadata with the keys set in it. """ d = dict(*args, **kwargs) - d.copy() - existing = self._data.copy() - + existing = copy.deepcopy(self._data) existing.update(d) - return UserMetadata(existing, shape=copy.copy(self._shape)) + + return UserMetadata(existing, shape=copy.deepcopy(self._shape)) def namespaces(self): return [] From 20e3048bca6c38aec08f23ddf6fdd220ee6d4935 Mon Sep 17 00:00:00 2001 From: Harrison Cook Date: Wed, 30 Apr 2025 14:23:10 +0000 Subject: [PATCH 5/5] Address second comments --- src/earthkit/data/utils/metadata/dict.py | 2 +- .../array_fieldlist/test_array_field_usermetadata.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py index 9cb65e388..7623750cb 100644 --- a/src/earthkit/data/utils/metadata/dict.py +++ b/src/earthkit/data/utils/metadata/dict.py @@ -471,7 +471,7 @@ def override(self, *args, **kwargs): d = dict(*args, **kwargs) existing = copy.deepcopy(self._data) existing.update(d) - + return UserMetadata(existing, shape=copy.deepcopy(self._shape)) def namespaces(self): diff --git a/tests/array_fieldlist/test_array_field_usermetadata.py b/tests/array_fieldlist/test_array_field_usermetadata.py index be860a2b9..ff1774e40 100644 --- a/tests/array_fieldlist/test_array_field_usermetadata.py +++ b/tests/array_fieldlist/test_array_field_usermetadata.py @@ -9,6 +9,7 @@ # nor does it submit to any jurisdiction. # +import copy import datetime import numpy as np @@ -94,20 +95,24 @@ def test_array_field_usermetadata_geom(_kwargs): ], ) def test_array_field_usermetadata_override(initial, update, expected): + + initial_copied = copy.deepcopy(initial) + update_copied = copy.deepcopy(update) + meta = UserMetadata(initial) new_meta = meta.override(update) - _ = meta.override(**update) + _ = meta.override(**update) # Check that the override method works with keyword arguments # Check that the updated metadata matches the expected result for k, v in expected.items(): assert new_meta[k] == v # Ensure the original metadata remains unchanged - for k, v in initial.items(): + for k, v in initial_copied.items(): assert meta[k] == v # Check that the updated metadata contains all expected keys - for k in update: + for k in update_copied: if k in initial: assert meta[k] == initial[k] else: