diff --git a/docs/source/how-tos/grib/grib_modify_metadata.ipynb b/docs/source/how-tos/grib/grib_modify_metadata.ipynb
index 4c3ae7a0..053fd925 100644
--- a/docs/source/how-tos/grib/grib_modify_metadata.ipynb
+++ b/docs/source/how-tos/grib/grib_modify_metadata.ipynb
@@ -45,7 +45,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "6bb5a5212f7742f6bd69894ec6a167b3",
+ "model_id": "2b3ffcefbc0b4828a3429d94ea98cc8c",
"version_major": 2,
"version_minor": 0
},
@@ -182,7 +182,9 @@
"tags": []
},
"source": [
- "A field can be modified by using :py:meth:`~earthkit.data.core.field.Field.set`. It will create a new field with updated metadata."
+ "A field can be modified by using :py:meth:`~earthkit.data.core.field.Field.set`. It will create a new field with updated metadata.\n",
+ "\n",
+ "The preferred way is to use the high-level field metadata keys whenever possible."
]
},
{
@@ -263,24 +265,23 @@
]
},
{
- "cell_type": "raw",
- "id": "296ef80e-510f-482c-bba3-99f353b5d4b5",
+ "cell_type": "markdown",
+ "id": "f4cb2de6-d5e6-4ca0-8cdb-63ae517919c8",
"metadata": {
"editable": true,
- "raw_mimetype": "text/restructuredtext",
"slideshow": {
"slide_type": ""
},
"tags": []
},
"source": [
- "Only the field component metadata keys can be used in :py:meth:`~earthkit.data.core.field.Field.set` and raw metadata keys are not allowed to use. E.g. since the field was created from GRIB data it has the raw (GRIB) metadata key ``metadata.shortName`` but we cannot set it. If you need to change the GRIB metadata see the \"Changing raw GRIB metadata\" section below."
+ "If you do need to use raw (ecCodes GRIB) metadata keys it is also possible (it only works for fields created from GRIB data)."
]
},
{
"cell_type": "code",
"execution_count": 4,
- "id": "799fcb48-8860-442c-86dc-16054146ef19",
+ "id": "6714df1d-42e9-4ff7-8a08-9563999503d7",
"metadata": {
"editable": true,
"slideshow": {
@@ -290,20 +291,162 @@
},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "t\n",
- "'Key metadata.shortName cannot be set on the field.'\n"
- ]
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " parameter.variable | \n",
+ " time.valid_datetime | \n",
+ " time.base_datetime | \n",
+ " time.step | \n",
+ " vertical.level | \n",
+ " vertical.level_type | \n",
+ " ensemble.member | \n",
+ " geography.grid_type | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " u | \n",
+ " 2018-08-01 12:00:00 | \n",
+ " 2018-08-01 12:00:00 | \n",
+ " 0 days | \n",
+ " 500 | \n",
+ " pressure | \n",
+ " 0 | \n",
+ " regular_ll | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n",
+ "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n",
+ "\n",
+ " vertical.level vertical.level_type ensemble.member geography.grid_type \n",
+ "0 500 pressure 0 regular_ll "
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
}
],
"source": [
- "print(f.get(\"metadata.shortName\"))\n",
- "try:\n",
- " f.set({\"metadata.shortName\": \"u\"})\n",
- "except Exception as e:\n",
- " print(e)"
+ "f1 = f.set({\"metadata.shortName\": \"u\", \"metadata.level\": 500})\n",
+ "f1.ls()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "424f1578-1f89-431e-8c62-ecae8194d348",
+ "metadata": {},
+ "source": [
+ "The two types of keys can be mixed. In this case the rule is that the high level keys are applied first, followed by the raw keys (prefixed with metadata)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "942d9709-6dde-4b30-a558-79cbac3571f8",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " parameter.variable | \n",
+ " time.valid_datetime | \n",
+ " time.base_datetime | \n",
+ " time.step | \n",
+ " vertical.level | \n",
+ " vertical.level_type | \n",
+ " ensemble.member | \n",
+ " geography.grid_type | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " u | \n",
+ " 2018-08-01 12:00:00 | \n",
+ " 2018-08-01 12:00:00 | \n",
+ " 0 days | \n",
+ " 500 | \n",
+ " pressure | \n",
+ " 0 | \n",
+ " regular_ll | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " parameter.variable time.valid_datetime time.base_datetime time.step \\\n",
+ "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n",
+ "\n",
+ " vertical.level vertical.level_type ensemble.member geography.grid_type \n",
+ "0 500 pressure 0 regular_ll "
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f1 = f.set({\"parameter.variable\": \"u\", \"parameter.units\": \"m/s\", \"metadata.level\": 500})\n",
+ "f1.ls()"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "id": "c0465c27-a52c-4b3a-a3c1-aec822b85900",
+ "metadata": {
+ "editable": true,
+ "raw_mimetype": "text/x-rst",
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "source": [
+ "Please note, there is an important GRIB related difference between using the high-level keys and the raw ecCodes GRIB keys in :py:meth:`~earthkit.data.core.field.Field.set`. Please see the \"Modified fields and the associated GRIB message\" chapter below for details."
]
},
{
@@ -336,7 +479,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"id": "77201dea-07e3-4e44-93d6-3220f249ea83",
"metadata": {
"editable": true,
@@ -401,7 +544,7 @@
"0 1000 pressure 0 regular_ll "
]
},
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -429,7 +572,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"id": "30a2c42b-56b9-42a4-ac22-f78d0bfe6c44",
"metadata": {
"editable": true,
@@ -494,7 +637,7 @@
"0 1000 pressure 0 regular_ll "
]
},
- "execution_count": 6,
+ "execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@@ -514,7 +657,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 8,
"id": "b384219d-0d52-4753-801e-39c6879754da",
"metadata": {},
"outputs": [
@@ -573,7 +716,7 @@
"0 1000 pressure 0 regular_ll "
]
},
- "execution_count": 7,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -608,7 +751,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 9,
"id": "e05d2d32-6e0a-41b7-bc6f-889b01856884",
"metadata": {
"editable": true,
@@ -673,7 +816,7 @@
"0 1000 pressure 0 regular_ll "
]
},
- "execution_count": 8,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -693,7 +836,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 10,
"id": "758a4ee9-23a9-45ce-91d7-9af9d1c6a930",
"metadata": {},
"outputs": [
@@ -730,7 +873,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 11,
"id": "f01193d1-cfbb-42fb-8f73-b11ab0b1737f",
"metadata": {},
"outputs": [
@@ -789,7 +932,7 @@
"0 500 pressure 0 regular_ll "
]
},
- "execution_count": 10,
+ "execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@@ -828,7 +971,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 12,
"id": "e403ad78-821d-4b14-8cae-4b2afef55093",
"metadata": {
"editable": true,
@@ -844,7 +987,7 @@
"b'GRIB\\x00\\x00\\x96\\x01\\x00\\x00'"
]
},
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
@@ -856,49 +999,73 @@
{
"cell_type": "markdown",
"id": "d9aa4f53-01e1-46de-a0c1-ebb87970cf38",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"source": [
- "Having modified the field metadata this GRIB message is not updated and we cannot access it any longer in the new field. The same is true for any raw GRIB metadata."
+ "Having modified the high-level field metadata this GRIB message is not updated and out of sync with the high-level field components. As a result, we cannot access it any longer in the new field. The same is true for any raw ecCodes GRIB metadata."
]
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 13,
"id": "8cddbcda-ea5c-4482-baa7-3535f94752a6",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [],
"source": [
"f1 = f.set({\"vertical.level\": 500})\n",
- "f1.message()"
+ "f1.message() # returns None"
]
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 14,
"id": "8750f04b-b106-493b-96ad-2c02c413137d",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [],
"source": [
- "f1.get(\"metadata.shortName\")"
+ "f1.get(\"metadata.level\") # returns None"
]
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 15,
"id": "5faa2075-b6ae-48a1-a07b-0b83055459e2",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "'Key metadata.shortName not found in field'\n"
+ "'Key metadata.level not found in field'\n"
]
}
],
"source": [
"try:\n",
- " f1.metadata(\"shortName\")\n",
+ " f1.metadata(\"level\")\n",
"except KeyError as e:\n",
" print(e)"
]
@@ -920,7 +1087,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 16,
"id": "5a2a63ec-0d95-4b3e-ba13-039f09234c81",
"metadata": {
"editable": true,
@@ -933,22 +1100,22 @@
{
"data": {
"text/plain": [
- "['t', 500]"
+ "500"
]
},
- "execution_count": 15,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"f1 = f1.sync()\n",
- "f1.get([\"metadata.shortName\", \"metadata.level\"])"
+ "f1.get(\"metadata.level\")"
]
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 17,
"id": "f17ffe64-45d4-4bdc-a174-26e8a4fa28d7",
"metadata": {
"editable": true,
@@ -961,16 +1128,16 @@
{
"data": {
"text/plain": [
- "['t', 500]"
+ "500"
]
},
- "execution_count": 16,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "f1.metadata([\"shortName\", \"level\"])"
+ "f1.metadata(\"level\")"
]
},
{
@@ -978,19 +1145,48 @@
"id": "af6311d3-74b0-4793-976e-d02a907f5113",
"metadata": {
"editable": true,
- "raw_mimetype": "text/restructuredtext",
+ "raw_mimetype": "text/x-rst",
"slideshow": {
"slide_type": ""
},
"tags": []
},
"source": [
- "Alternatively, if your workflow is strictly GRIB-bound you can carry out the filed modification via the :py:class:`~earthkit.data.encoders.grib.GribEncoder` as shown in the next chapter. "
+ "Alternatively, we can use the ``sync=True`` kwarg in :py:meth:`~earthkit.data.core.field.Field.set` to execute the syncing as part of the setting process."
]
},
{
- "cell_type": "markdown",
- "id": "39666d1c-7db3-4e97-9cfc-55b3b8d71f7d",
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "68a24731-785a-453d-a809-b901f0bd7dae",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "b'GRIB\\x00\\x00\\x96\\x01\\x00\\x00'"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f1 = f.set({\"vertical.level\": 500}, sync=True)\n",
+ "f1.message()[:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "968ee5be-a6ff-4e41-964b-fc53cb45854e",
"metadata": {
"editable": true,
"slideshow": {
@@ -998,30 +1194,41 @@
},
"tags": []
},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "500"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "## Changing raw GRIB metadata"
+ "f1.get(\"metadata.level\")"
]
},
{
"cell_type": "raw",
- "id": "89482515-0cf7-44ce-a267-c78e8bd0a6a6",
+ "id": "60b11297-7499-445d-9b71-2d59709aed25",
"metadata": {
"editable": true,
- "raw_mimetype": "text/restructuredtext",
+ "raw_mimetype": "text/x-rst",
"slideshow": {
"slide_type": ""
},
"tags": []
},
"source": [
- "Currently, changing the (raw) GRIB metadata in a field requires the usage of a :py:class:`~earthkit.data.encoders.grib.GribEncoder`. \n",
- "When we call its :py:meth:`~earthkit.data.encoders.grib.GribEncoder.encode` method it will clone the underlying GRIB message, set the GRIB metadata on it and return an object that can be converted to a field."
+ "Obviously, when only raw metadata keys are used in :func:`~earthkit.data.core.field.Field.set` there is no need for syncing."
]
},
{
"cell_type": "code",
- "execution_count": 17,
- "id": "82e780fe-3539-4af9-ad4f-585739577dcc",
+ "execution_count": 20,
+ "id": "6ced9bcd-8727-49f0-a48e-fb986292c516",
"metadata": {
"editable": true,
"slideshow": {
@@ -1032,75 +1239,24 @@
"outputs": [
{
"data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " parameter.variable | \n",
- " time.valid_datetime | \n",
- " time.base_datetime | \n",
- " time.step | \n",
- " vertical.level | \n",
- " vertical.level_type | \n",
- " ensemble.member | \n",
- " geography.grid_type | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " u | \n",
- " 2018-08-01 12:00:00 | \n",
- " 2018-08-01 12:00:00 | \n",
- " 0 days | \n",
- " 1000 | \n",
- " pressure | \n",
- " 0 | \n",
- " regular_ll | \n",
- "
\n",
- " \n",
- "
\n",
- "
"
- ],
"text/plain": [
- " parameter.variable time.valid_datetime time.base_datetime time.step \\\n",
- "0 u 2018-08-01 12:00:00 2018-08-01 12:00:00 0 days \n",
- "\n",
- " vertical.level vertical.level_type ensemble.member geography.grid_type \n",
- "0 1000 pressure 0 regular_ll "
+ "b'GRIB\\x00\\x00\\x96\\x01\\x00\\x00'"
]
},
- "execution_count": 17,
+ "execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "encoder = ekd.create_encoder(\"grib\")\n",
- "r = encoder.encode(template=f, metadata={\"shortName\": \"u\"})\n",
- "f1 = r.to_field()\n",
- "f1.ls()"
+ "f1 = f.set({\"metadata.level\": 500})\n",
+ "f1.message()[:10]"
]
},
{
"cell_type": "code",
- "execution_count": null,
- "id": "60b11297-7499-445d-9b71-2d59709aed25",
+ "execution_count": 21,
+ "id": "7496dea5-efae-4ad8-bbec-d4990e571164",
"metadata": {
"editable": true,
"slideshow": {
@@ -1108,8 +1264,21 @@
},
"tags": []
},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "500"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f1.get(\"metadata.level\")"
+ ]
}
],
"metadata": {
diff --git a/docs/source/how-tos/grib/grib_modify_values.ipynb b/docs/source/how-tos/grib/grib_modify_values.ipynb
index a63b4f54..ff8fca77 100644
--- a/docs/source/how-tos/grib/grib_modify_values.ipynb
+++ b/docs/source/how-tos/grib/grib_modify_values.ipynb
@@ -39,7 +39,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "0a3b6df9fe9e40d8a214a4d43332f8fa",
+ "model_id": "8789aed77cf04658bed3995c190c02eb",
"version_major": 2,
"version_minor": 0
},
@@ -534,14 +534,20 @@
"tags": []
},
"source": [
- "If the metadata is also modified the associated GRIB message is not updated and we cannot access it any longer in the new field. The same is true for any raw GRIB metadata."
+ "If the high level metadata is also modified the associated GRIB message is not updated and we cannot access it any longer in the new field. The same is true for any raw GRIB metadata."
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "a29ae7ac-4617-48c8-9176-98ad89c1ceec",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [],
"source": [
"f = fl[0]\n",
@@ -551,22 +557,161 @@
{
"cell_type": "code",
"execution_count": 14,
+ "id": "0982dbc2-df59-40b6-912f-70c4e745f870",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "np.float64(273.5641784667969)"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# this works\n",
+ "f1.values[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
"id": "b21a3608-693d-43ac-b851-ebd9998306c6",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [],
"source": [
- "f1.message()"
+ "f1.message() # returns None"
]
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 16,
"id": "74cb432d-797a-4685-9415-fa2583f67e4e",
- "metadata": {},
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
"outputs": [],
+ "source": [
+ "f1.get(\"metadata.shortName\") # returns None"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "id": "39854f6d-e4ba-401d-98d2-608da16c356f",
+ "metadata": {
+ "editable": true,
+ "raw_mimetype": "text/x-rst",
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "source": [
+ "If we want to keep a valid associated GRIB message in the modified field we need to call :func:`~earthkit.data.core.field.Field.sync`. This will create a new GRIB handle, update the relevant metadata in it and create a new field out of it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "03d40091-9d08-4e29-9856-cc3ebeaf055a",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "b'GRIB\\x00\\x00\\x96\\x01\\x00\\x00'"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f1 = f1.sync()\n",
+ "f1.message()[:10]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "22881faf-e979-4a06-b6b1-9ffce41db759",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'t'"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"f1.get(\"metadata.shortName\")"
]
+ },
+ {
+ "cell_type": "raw",
+ "id": "73eec817-5a3b-4880-8609-b2b2eaa67b73",
+ "metadata": {
+ "editable": true,
+ "raw_mimetype": "text/x-rst",
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "source": [
+ "See the :ref:`/how-tos/grib/grib_modify_metadata.ipynb` example for details."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0a45856c-902c-47ee-ac05-75d0f282944e",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": []
}
],
"metadata": {
diff --git a/src/earthkit/data/core/field.py b/src/earthkit/data/core/field.py
index 5c4d5ae7..337ea5b7 100644
--- a/src/earthkit/data/core/field.py
+++ b/src/earthkit/data/core/field.py
@@ -1323,17 +1323,18 @@ def set(self, *args, sync=False, **kwargs):
>>> field.set({"parameter.variable": "t", "vertical.level": 1000})
New data values can be set by using the "data" or "values" key with the new values
- as a value. For example,
+ as a value. For example, both these calls are equivalent and will replace the data
+ values in the field with the values in ``new_values_array``:
>>> field.set(data=new_values_array)
+ >>> field.set(values=new_values_array)
- will replace the data values in the field with the values in ``new_values_array``.
-
- Only high-level metadata keys (and "data" or "values") are allowed here, i.e. keys that
- belong to a component. Modifying raw metadata keys is not allowed and we cannot use them
- in :meth:`set` with or without the "metadata." prefix. For example, although
- in fields generated from GRIB we can use the "metadata.shortName" key in the :meth:`get`
- method to access the "shortName" key we cannot use it in :meth:`set`.
+ Preferably, high-level metadata keys (and "data" or "values") should be used in :meth:`set`
+ i.e. keys that belong to a component. However, modifying raw metadata keys is also
+ allowed, but they must be used with the "metadata." prefix just like in :meth:`get`.
+ Mixing high-level and raw metadata keys is allowed as well. In this keys the high-level
+ keys are set first and then the raw metadata keys. Please see the Notes section below
+ for the implications of modifying high-level metadata keys when the field has raw metadata.
Entire components can be set by using the component name as a key and the component
object or the equivalent dict as a value. For example,
@@ -1376,30 +1377,47 @@ def set(self, *args, sync=False, **kwargs):
Notes
-----
- When the field is created from a GRIB message, calling :meth:`set` copies the associated
- GRIB message into the new field without any modifications. Since it is now out of sync with the
- new field's components, the new field will not provide access to any GRIB metadata
- neither via :meth:`get` nor via :meth:`metadata`. Additionally, when calling
- :meth:`message` on the new field, None is returned. (Use :meth:`sync` to synchronize the
- associated GRIB message to the new field and expose the GRIB metadata keys again).
+ When the field has raw metadata (to date it is only the case for fields created from
+ GRIB messages) calling :meth:`set` with high-level metadata keys will not update the raw
+ metadata in the new field and it is now out of sync
+ with the new field's components and cannot be accessed anymore via :meth:`get` or
+ :meth:`metadata`. The :meth:`message` method will not work either. To access the raw
+ metadata keys again, use :meth:`sync` to synchronize
+ the raw metadata to the new field. Alternatively, using the ``sync=True`` option in :meth:`set`
+ will try to do this automatically. Please note, it might not be possible to synchronize
+ the new field to the raw metadata and an exception can be raised in this case.
+
+ The following example using GRIB data shows how modifying high-level metadata keys results in
+ a new field where the raw metadata is out of sync and cannot be accessed anymore
+ until :meth:`sync` is called:
>>> import earthkit.data as ekd
>>> fl = ekd.from_source("sample", "test.grib").to_fieldlist()
>>> f = fl[0]
>>> f1 = f.set({"parameter.variable": "msl", "parameter.units": "Pa"})
+ >>> f1.get("parameter.variable")
+ 'msl'
>>> f1.get("metadata.shortName")
None
>>> f1.metadata("shortName")
KeyError: 'metadata.shortName' not found in field
>>> f1.message()
None
+ >>> f1 = f1.sync()
+ >>> f1.get("metadata.shortName")
+ 'msl'
+ >>> f1.metadata("shortName")
+ 'msl'
+ >>> f1.message()[:10]
+
- However, if only the labels or the values are set (the latter via the "data" or "values" keys), the new
- field returned by :meth:`set` is still linked to the original GRIB message and the GRIB specific keys
- in the raw metadata are still accessible. If the values were modified, :meth:`message` will return
- the original GRIB message updated with the modified data values.
+ When only the labels or the values are set (the latter via the "data" or "values" keys),
+ the new field returned by :meth:`set` the original raw metadata is still valid and the
+ ecCodes GRIB keys are still accessible. There is no need to synchronize the raw metadata in
+ this case. If the values were modified, :meth:`message` will return the original GRIB message
+ updated with the modified data values.
- >>> import earthkit.data as ekd
+ >>> import earthkit.data as ekd
>>> fl = ekd.from_source("sample", "test.grib").to_fieldlist()
>>> f = fl[0]
>>> f1 = f.set(values=f.values()+1)
@@ -1445,6 +1463,7 @@ def set(self, *args, sync=False, **kwargs):
return self
_components = dict()
+ _raw = dict()
for k, v in kwargs.items():
if k in self._components:
_components[k] = v
@@ -1454,7 +1473,12 @@ def set(self, *args, sync=False, **kwargs):
if key_name is not None and key_name != "":
_kwargs[component_name][key_name] = v
else:
- raise KeyError(f"Key {k} cannot be set on the field.")
+ raise KeyError(f"Invalid key={k} specified.")
+ elif component_name == _METADATA:
+ if key_name is not None and key_name != "":
+ _raw[key_name] = v
+ else:
+ raise KeyError(f"Invalid key={k} specified.")
else:
raise KeyError(f"Key {k} cannot be set on the field.")
@@ -1470,12 +1494,19 @@ def set(self, *args, sync=False, **kwargs):
s = component.set(**v)
_components[component_name] = s
+ f_new = self
if _components:
f_new = self._from_set(**_components)
+
+ if _raw:
+ return f_new._set_metadata(_raw)
+
+ if _components:
if sync:
- f_new = f_new.sync()
+ return f_new.sync()
return f_new
- elif kwargs:
+
+ if kwargs:
raise ValueError("No valid keys to set in the field.")
return None
@@ -1524,15 +1555,7 @@ def sync(self):
>>> f2.metadata("shortName")
'msl'
"""
- if self._get_grib() and self._private and "_metadata" in self._private:
- from earthkit.data.encoders.grib import GribEncoder
-
- encoder = GribEncoder()
- f = encoder.encode(data=self).to_field()
- if self.labels:
- f = f.set(labels=self.labels)
- return f
- return self
+ return self._sync_metadata()
def to_target(self, target, *args, **kwargs):
r"""Write the field into a target object.
@@ -1894,6 +1917,23 @@ def _get_grib_context(self, context):
for m in self._components.values():
m.get_grib_context(context)
+ def _set_metadata(self, md):
+ g = self._get_grib()
+ if g is not None:
+ from earthkit.data.encoders.grib import GribEncoder
+
+ encoder = GribEncoder()
+ return encoder.encode(data=self, metadata=md).to_field()
+ return self
+
+ def _sync_metadata(self):
+ if self._get_grib() and self._private and "_metadata" in self._private:
+ from earthkit.data.encoders.grib import GribEncoder
+
+ encoder = GribEncoder()
+ return encoder.encode(data=self).to_field()
+ return self
+
@normalise("time.valid_datetime", "date")
@normalise("time.base_datetime", "date")
@normalise("time.forecast_reference_time", "date")
diff --git a/src/earthkit/data/encoders/grib.py b/src/earthkit/data/encoders/grib.py
index 4cf4e958..63b095e2 100644
--- a/src/earthkit/data/encoders/grib.py
+++ b/src/earthkit/data/encoders/grib.py
@@ -30,15 +30,20 @@
class GribEncodedData(EncodedData):
"""The object representing the encoded GRIB message."""
- def __init__(self, handle):
+ def __init__(self, handle, template_field=None):
"""Initialize the GribEncodedData object.
Parameters
----------
handle: GribCodesHandle
The handle to the GRIB message.
+ template_field: Field, optional
+ A template field to use when converting to a Field object. This is
+ used to provide extra field data (e.g. labels) that cannot be obtained
+ from the GRIB message itself.
"""
self.handle = handle
+ self._template_field = template_field
def to_bytes(self):
"""Return the GRIB message as bytes."""
@@ -80,7 +85,7 @@ def to_field(self):
"""Convert the GRIB message to a Field object."""
from earthkit.data.field.grib.create import create_grib_field_from_message
- return create_grib_field_from_message(self.to_bytes())
+ return create_grib_field_from_message(self.to_bytes(), template_field=self._template_field)
class Combined:
@@ -140,15 +145,15 @@ def __init__(self, template=None):
self.template = self.handle_from_template(template, clone=False)
self._bbox = {}
- def make(self, values=None, metadata=None, template=None, field_metadata=None):
+ def make(self, values_shape=None, metadata=None, template=None, field_metadata=None):
"""Create a new GribCodesHandle from a template, field or metadata.
May modify existing metadata
Parameters
----------
- values: numpy.ndarray, optional
- The values to encode
+ values_shape: tuple, optional
+ The shape of the values to encode
metadata: dict, optional
Metadata to encode
template: GribCoder, optional
@@ -159,21 +164,23 @@ def make(self, values=None, metadata=None, template=None, field_metadata=None):
if template is None:
template = self.template
- handle = self.handle_from_template(template, field_metadata=field_metadata, clone=True)
+ handle = self.handle_from_template(
+ template, values_shape=values_shape, field_metadata=field_metadata, clone=True
+ )
if handle is not None:
self.update_metadata_from_template(metadata, template, handle)
if handle is None:
- if values is None:
+ if values_shape is None:
raise ValueError("No values to encode")
if field_metadata:
raise ValueError("Cannot provide field_metadata without a template or handle")
- handle = self.handle_from_metadata(values, metadata, _COMPULSORY)
+ handle = self.handle_from_metadata(values_shape, metadata, _COMPULSORY)
return handle
@staticmethod
- def handle_from_template(template, field_metadata=None, clone=True):
+ def handle_from_template(template, values_shape=None, field_metadata=None, clone=True):
handle = None
if template is not None:
from earthkit.data.core.field import Field
@@ -191,7 +198,7 @@ def _result(handle):
if isinstance(template, Field):
if field_metadata:
template = template.set(**field_metadata)
- return GribHandleMaker.handle_from_field(template)
+ return GribHandleMaker.handle_from_field(template, values_shape=values_shape)
# GribMetadata or GribHandle
elif hasattr(template, "handle"):
handle = template.handle
@@ -225,7 +232,7 @@ def _result(handle):
return None
@staticmethod
- def handle_from_field(field):
+ def handle_from_field(field, values_shape=None):
r = {}
field = field.sync()
field._get_grib_context(r)
@@ -233,18 +240,21 @@ def handle_from_field(field):
if handle is not None:
handle = handle.clone()
+ if values_shape is None:
+ if "values" in r:
+ handle.set_values(r["values"])
return handle
- def handle_from_metadata(self, values, metadata, compulsory):
+ def handle_from_metadata(self, values_shape, metadata, compulsory):
from earthkit.data.readers.grib.handle import GribCodesHandle # Lazy loading of eccodes
- if len(values.shape) == 1:
- sample = self._gg_field(values, metadata)
- elif len(values.shape) == 2:
- sample = self._ll_field(values, metadata)
+ if len(values_shape) == 1:
+ sample = self._gg_field(values_shape, metadata)
+ elif len(values_shape) == 2:
+ sample = self._ll_field(values_shape, metadata)
else:
- raise ValueError(f"Invalid shape {values.shape} for GRIB, must be 1 or 2 dimension ")
+ raise ValueError(f"Invalid shape {values_shape} for GRIB, must be 1 or 2 dimension ")
metadata.setdefault("bitsPerValue", 16)
metadata["scanningMode"] = 0
@@ -281,8 +291,8 @@ def update_metadata_from_template(self, metadata, template, handle):
else:
bpv = template_md.get("bitsPerValue", default=None)
- def _ll_field(self, values, metadata):
- Nj, Ni = values.shape
+ def _ll_field(self, values_shape, metadata):
+ Nj, Ni = values_shape
metadata["Nj"] = Nj
metadata["Ni"] = Ni
@@ -319,7 +329,7 @@ def _ll_field(self, values, metadata):
return f"regular_ll_{levtype}_grib{edition}"
- def _gg_field(self, values, metadata):
+ def _gg_field(self, values_shape, metadata):
GAUSSIAN = {
6114: (32, False),
13280: (48, False),
@@ -342,7 +352,7 @@ def _gg_field(self, values, metadata):
20696844: (2000, False),
}
- n = len(values)
+ n = values_shape[0]
if n not in GAUSSIAN:
raise ValueError(f"Unsupported GAUSSIAN grid. Number of grid points {n:,}")
N, octahedral = GAUSSIAN[n]
@@ -449,6 +459,13 @@ def __init__(self, template=None, metadata=None, **kwargs):
self._bbox = {}
# the template is stored as a handle to be used as a basis for encoding,
# (when available)
+ self._template_field = None
+ if template is not None:
+ from earthkit.data import Field
+
+ if isinstance(template, Field):
+ self._template_field = template
+
self.template = GribHandleMaker.handle_from_template(self.template, clone=False)
@normalise_grib_keys
@@ -556,8 +573,15 @@ def encode(
- :ref:`/how-tos/target/grib_encoder.ipynb`
"""
+ template_field = None
if template is None:
template = self.template
+ template_field = self._template_field
+ else:
+ from earthkit.data import Field
+
+ if isinstance(template, Field):
+ template_field = template
if data is not None and values is not None and template:
raise ValueError("Cannot provide data, values and template together")
@@ -602,17 +626,43 @@ def encode(
hints = {"path_allowed": path_allowed}
if data is not None:
+ from earthkit.data import Field
from earthkit.data.data.wrappers import from_object
+ if template_field is None:
+ template_field = data if isinstance(data, Field) else None
+
data = from_object(data)
return data._encode(
- self, hints=hints, target=target, template=template, field_metadata=field_metadata, **kwargs
+ self,
+ hints=hints,
+ target=target,
+ template=template,
+ template_field=template_field,
+ field_metadata=field_metadata,
+ **kwargs,
)
else:
handle = self._get_handle(
- template=template, values=values, metadata=metadata, field_metadata=field_metadata
+ template=template,
+ values_shape=values.shape if values is not None else None,
+ metadata=metadata,
+ field_metadata=field_metadata,
)
- return self._make_message(handle, **kwargs)
+ new_handle = self._make_new_handle(handle, **kwargs)
+ return GribEncodedData(new_handle, template_field=template_field)
+
+ def _template(self, data, template):
+ if template is None:
+ template = self.template
+ template_field = self._template_field
+ else:
+ from earthkit.data import Field
+
+ if isinstance(template, Field):
+ template_field = template
+
+ return template, template_field
def _has_standard_date_input(self, d):
for v in d:
@@ -627,7 +677,9 @@ def _has_standard_date_input(self, d):
def _encode(self, data, *, target=None, **kwargs):
raise NotImplementedError
- def _encode_field(self, field, *, target=None, values=None, template=None, metadata=None, **kwargs):
+ def _encode_field(
+ self, field, *, target=None, values=None, template=None, template_field=None, metadata=None, **kwargs
+ ):
# check if the field is already encoded in the desired format
r = {}
@@ -645,7 +697,7 @@ def _encode_field(self, field, *, target=None, values=None, template=None, metad
metadata = r
if field_values is None and values is None and template is None and not metadata:
- return GribEncodedData(handle)
+ return GribEncodedData(handle, template_field=template_field)
# set bitspervalue
if "bitsPerValue" not in metadata:
@@ -671,9 +723,13 @@ def _encode_field(self, field, *, target=None, values=None, template=None, metad
if template is None:
template = handle
- handle = self._get_handle(values=values, metadata=metadata, template=template)
+ handle = self._get_handle(
+ values_shape=values.shape if values is not None else None, metadata=metadata, template=template
+ )
- return self._make_message(handle, values=values, metadata=metadata, **kwargs)
+ new_handle = self._make_new_handle(handle, values=values, metadata=metadata, **kwargs)
+
+ return GribEncodedData(new_handle, template_field=template_field)
def _encode_fieldlist(self, fs, *, target=None, **kwargs):
for f in fs:
@@ -699,7 +755,7 @@ def _encode_path(self, path_info, *, target=None, **kwargs):
else:
return None
- def _make_message(
+ def _make_new_handle(
self, handle, values=None, check_nans=True, metadata=None, missing_value=9999, can_infer_time=False
):
if handle is None:
@@ -782,7 +838,8 @@ def _make_message(
if values is not None:
handle.set_values(values)
- return GribEncodedData(handle)
+ return handle
+ # return GribEncodedData(handle)
def _update_metadata_from_field(self, field, metadata):
if "stepRange" in metadata:
diff --git a/tests/encoders/test_grib_encoder.py b/tests/encoders/test_grib_encoder.py
index be8caf14..f8ae5e59 100644
--- a/tests/encoders/test_grib_encoder.py
+++ b/tests/encoders/test_grib_encoder.py
@@ -17,6 +17,9 @@
from earthkit.data import create_encoder, from_source
from earthkit.data.utils.testing import earthkit_examples_file
+_LABELS = {"my_label": "my_val"}
+_LABELS_SET_ARGS = {"labels.my_label": "my_val"}
+
@pytest.mark.parametrize("_args,_kwargs", [(("",), {}), ((), {"data": ""}), ((), {"template": ""})])
def test_grib_encoder_field_1(_args, _kwargs):
@@ -38,28 +41,31 @@ def test_grib_encoder_field_1(_args, _kwargs):
@pytest.mark.parametrize("init_encoder", [None, ["template"]])
-@pytest.mark.parametrize("template_arg", ["field", "message", "handle", "raw_handle"])
+@pytest.mark.parametrize("template_arg", ["field", "field_labels", "message", "handle", "raw_handle"])
def test_grib_encoder_field_template_only(init_encoder, template_arg):
fl = from_source("file", earthkit_examples_file("test.grib")).to_fieldlist()
template = fl[1]
if template_arg == "message":
- template_arg = template.message()
+ template_arg_d = template.message()
elif template_arg == "field":
- template_arg = template
+ template_arg_d = template
+ elif template_arg == "field_labels":
+ template = template.set(_LABELS_SET_ARGS)
+ template_arg_d = template
elif template_arg == "handle":
- template_arg = template._get_grib().handle
+ template_arg_d = template._get_grib().handle
elif template_arg == "raw_handle":
# this is the clone of the raw handle
- template_arg = template._get_grib().handle._raw_handle()
+ template_arg_d = template._get_grib().handle._raw_handle()
else:
raise ValueError(f"Invalid template_arg: {template_arg}")
assert template.get("parameter.variable") == "msl"
encoder_kwargs = {}
- encode_kwargs = {"template": template_arg}
+ encode_kwargs = {"template": template_arg_d}
if init_encoder is not None:
for key in init_encoder:
if key in encode_kwargs:
@@ -75,10 +81,12 @@ def test_grib_encoder_field_template_only(init_encoder, template_arg):
assert template.message() == f_r.message()
assert np.allclose(template.values, f_r.values)
assert f_r.get("parameter.variable") == "msl"
+ if template_arg == "field_labels":
+ assert f_r.labels.to_dict() == _LABELS
@pytest.mark.parametrize("init_encoder", [None, ["template"]])
-@pytest.mark.parametrize("template_arg", ["field", "message", "handle", "raw_handle"])
+@pytest.mark.parametrize("template_arg", ["field", "field_labels", "message", "handle", "raw_handle"])
def test_grib_encoder_field_data_and_template(init_encoder, template_arg):
fl = from_source("file", earthkit_examples_file("test.grib")).to_fieldlist()
@@ -86,13 +94,16 @@ def test_grib_encoder_field_data_and_template(init_encoder, template_arg):
template = fl[1]
if template_arg == "message":
- template_arg = template.message()
+ template_arg_d = template.message()
elif template_arg == "field":
- template_arg = template
+ template_arg_d = template
+ elif template_arg == "field_labels":
+ template = template.set(_LABELS_SET_ARGS)
+ template_arg_d = template
elif template_arg == "handle":
- template_arg = template._get_grib().handle
+ template_arg_d = template._get_grib().handle
elif template_arg == "raw_handle":
- template_arg = template._get_grib().handle._raw_handle()
+ template_arg_d = template._get_grib().handle._raw_handle()
else:
raise ValueError(f"Invalid template_arg: {template_arg}")
@@ -100,7 +111,7 @@ def test_grib_encoder_field_data_and_template(init_encoder, template_arg):
assert template.get("parameter.variable") == "msl"
encoder_kwargs = {}
- encode_kwargs = {"template": template_arg}
+ encode_kwargs = {"template": template_arg_d}
if init_encoder is not None:
for key in init_encoder:
if key in encode_kwargs:
@@ -119,10 +130,12 @@ def test_grib_encoder_field_data_and_template(init_encoder, template_arg):
assert np.allclose(f.values, f_r.values)
assert f.get("parameter.variable") == "2t"
assert f_r.get("parameter.variable") == "msl"
+ if template_arg == "field_labels":
+ assert f_r.labels.to_dict() == _LABELS
@pytest.mark.parametrize("init_encoder", [None, ["template"]])
-@pytest.mark.parametrize("template_arg", ["field", "message", "handle", "raw_handle"])
+@pytest.mark.parametrize("template_arg", ["field", "field_labels", "message", "handle", "raw_handle"])
def test_grib_encoder_field_values_and_template(init_encoder, template_arg):
fl = from_source("file", earthkit_examples_file("test.grib")).to_fieldlist()
@@ -131,13 +144,16 @@ def test_grib_encoder_field_values_and_template(init_encoder, template_arg):
template = fl[1]
if template_arg == "message":
- template_arg = template.message()
+ template_arg_d = template.message()
elif template_arg == "field":
- template_arg = template
+ template_arg_d = template
+ elif template_arg == "field_labels":
+ template = template.set(_LABELS_SET_ARGS)
+ template_arg_d = template
elif template_arg == "handle":
- template_arg = template._get_grib().handle
+ template_arg_d = template._get_grib().handle
elif template_arg == "raw_handle":
- template_arg = template._get_grib().handle._raw_handle()
+ template_arg_d = template._get_grib().handle._raw_handle()
else:
raise ValueError(f"Invalid template_arg: {template_arg}")
@@ -145,7 +161,7 @@ def test_grib_encoder_field_values_and_template(init_encoder, template_arg):
assert template.get("parameter.variable") == "msl"
encoder_kwargs = {}
- encode_kwargs = {"template": template_arg}
+ encode_kwargs = {"template": template_arg_d}
if init_encoder is not None:
for key in init_encoder:
if key in encode_kwargs:
@@ -164,6 +180,8 @@ def test_grib_encoder_field_values_and_template(init_encoder, template_arg):
assert np.allclose(f.values + 1.0, f_r.values)
assert f.get("parameter.variable") == "2t"
assert f_r.get("parameter.variable") == "msl"
+ if template_arg == "field_labels":
+ assert f_r.labels.to_dict() == _LABELS
def test_grib_encoder_field_data_and_values_and_template():
@@ -390,3 +408,25 @@ def test_grib_encoder_field_mixed_metadata():
assert f_r.get("metadata.step") == 5
assert f_r.get("metadata.validityDate") == 19980502
assert f_r.get("metadata.validityTime") == 1100
+
+
+@pytest.mark.parametrize("_args,_kwargs", [(("",), {}), ((), {"data": ""}), ((), {"template": ""})])
+def test_grib_encoder_field_labels_1(_args, _kwargs):
+ f = from_source("file", earthkit_examples_file("test.grib")).to_fieldlist()[0]
+
+ f = f.set({"labels.my_label": "val"})
+
+ _args = tuple(f if v == "" else v for v in _args)
+ _kwargs = {k: (f if v == "" else v) for k, v in _kwargs.items()}
+
+ encoder = create_encoder("grib")
+ r = encoder.encode(*_args, **_kwargs)
+
+ assert r.to_bytes() == f.message()
+
+ f_r = r.to_field()
+ assert f is not f_r
+ assert f.message() == f_r.message()
+ assert np.allclose(f.values, f_r.values)
+ assert f.get("parameter.variable") == f_r.get("parameter.variable")
+ assert f.get("labels.my_label") == "val"
diff --git a/tests/grib/test_grib_set.py b/tests/grib/test_grib_set.py
index b8ca7f58..4301a674 100644
--- a/tests/grib/test_grib_set.py
+++ b/tests/grib/test_grib_set.py
@@ -148,15 +148,11 @@ def test_grib_set_fieldlist_detailed(fl_type):
@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
-def test_grib_set_combined(fl_type):
+def test_grib_set_field_metadata_and_values(fl_type):
ds_ori, _ = load_grib_data("test4.grib", fl_type)
vals_ori = ds_ori[0].values
- # ---------------
- # field
- # ---------------
-
f = ds_ori[0].set({
"values": vals_ori + 1,
"parameter.variable": "q",
@@ -184,7 +180,7 @@ def test_grib_set_combined(fl_type):
assert np.allclose(f_saved.values, vals_ori + 1)
# ---------------------
- # field - repeated use
+ # repeated use
# ---------------------
f = ds_ori[0].set({
@@ -207,9 +203,12 @@ def test_grib_set_combined(fl_type):
assert np.allclose(f.values, vals_ori + 2)
assert np.allclose(ds_ori[0].values, vals_ori)
- # ---------------
- # fieldlist
- # ---------------
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+def test_grib_set_fieldlist_metadata_and_values(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+
+ vals_ori = ds_ori[0].values
fields = []
for i in range(2):
@@ -447,6 +446,9 @@ def test_grib_set_field_sync(fl_type):
assert f1.get("labels.my_shape") == (181, 360)
assert f1.get("labels.my_name") == "t_500"
+ f2 = f1.sync()
+ assert f2 is f1
+
@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
# @pytest.mark.parametrize("fl_type", ["file"])
@@ -483,3 +485,226 @@ def test_grib_set_field_sync_kwarg(fl_type):
assert f1.get(("parameter.variable", "metadata.date")) == ("q", 20070101)
assert f1.get("labels.my_shape") == (181, 360)
assert f1.get("labels.my_name") == "t_500"
+
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+# @pytest.mark.parametrize("fl_type", ["file"])
+def test_grib_set_field_raw_detailed_1(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+
+ f = ds_ori[0].set({
+ "metadata.shortName": "q",
+ "metadata.level": 600,
+ "labels.my_shape": (181, 360),
+ "labels.my_name": "t_500",
+ })
+
+ assert f.get("parameter.variable") == "q"
+ assert f.get("metadata.shortName") == "q"
+ assert f.get("vertical.level") == 600
+ assert f.get("metadata.levelist") == 600
+ assert f.get(("metadata.date", "parameter.variable")) == (20070101, "q")
+ assert f.get(("parameter.variable", "metadata.date")) == ("q", 20070101)
+ assert f.get("labels.my_shape") == (181, 360)
+ assert f.get("labels.my_name") == "t_500"
+
+ # write back to grib
+ with temp_file() as tmp:
+ f = ds_ori[0].set({
+ "parameter.variable": "q",
+ "vertical.level": 600,
+ })
+
+ f.to_target("file", tmp)
+ f_saved = from_source("file", tmp).to_fieldlist()[0]
+ assert f_saved.get("parameter.variable") == "q"
+ assert f_saved.get("parameter.variable") == "q"
+ assert f_saved.get("metadata.shortName") == "q"
+ assert f_saved.get("vertical.level") == 600
+ assert f_saved.get("metadata.level") == 600
+ assert f_saved.get("metadata.levelist") == 600
+ assert f_saved.get("vertical.level_type") == "pressure"
+ assert f_saved.get("metadata.typeOfLevel") == "isobaricInhPa"
+
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+# @pytest.mark.parametrize("fl_type", ["file"])
+def test_grib_set_field_raw_detailed_2(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+
+ f = ds_ori[0].set({
+ "metadata.shortName": "q",
+ "metadata.level": 600,
+ "labels.my_shape": (181, 360),
+ "labels.my_name": "t_500",
+ })
+
+ f = f.set({
+ "metadata.shortName": "pt",
+ "metadata.level": 800,
+ })
+
+ assert f.get("parameter.variable") == "pt"
+ assert f.get("metadata.shortName") == "pt"
+ assert f.get("vertical.level") == 800
+ assert f.get("metadata.level") == 800
+ assert f.get("metadata.levelist") == 800
+ assert f.get(("metadata.date", "parameter.variable")) == (20070101, "pt")
+ assert f.get(("parameter.variable", "metadata.date")) == ("pt", 20070101)
+ assert f.get("labels.my_name") == "t_500"
+
+ f1 = f.sync()
+ assert f1.get("parameter.variable") == "pt"
+ assert f1.get("metadata.shortName") == "pt"
+ assert f1.get("vertical.level") == 800
+ assert f1.get("metadata.level") == 800
+ assert f1.get("metadata.levelist") == 800
+ assert f1.get("metadata.typeOfLevel") == "isobaricInhPa"
+ assert f1.get("vertical.level_type") == "pressure"
+ assert f1.get(("metadata.date", "parameter.variable")) == (20070101, "pt")
+ assert f1.get(("parameter.variable", "metadata.date")) == ("pt", 20070101)
+ assert f1.get("labels.my_name") == "t_500"
+
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+def test_grib_set_field_raw_metadata_and_values(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+
+ vals_ori = ds_ori[0].values
+
+ f = ds_ori[0].set({
+ "values": vals_ori + 1,
+ "metadata.shortName": "q",
+ "metadata.level": 600,
+ })
+
+ assert f.get("parameter.variable") == "q"
+ assert f.get("metadata.shortName") == "q"
+ assert f.get("vertical.level") == 600
+ assert f.get("metadata.levelist") == 600
+ assert f.get(("metadata.date", "parameter.variable")) == (20070101, "q")
+ assert f.get(("parameter.variable", "metadata.date")) == ("q", 20070101)
+ np.testing.assert_allclose(f.values, vals_ori + 1)
+ assert np.allclose(f.values, vals_ori + 1)
+ assert np.allclose(ds_ori[0].values, vals_ori)
+
+ # write back to grib
+ # we can only have ecCodes keys
+ with temp_file() as tmp:
+ f.to_target("file", tmp)
+ f_saved = from_source("file", tmp).to_fieldlist()[0]
+ assert f_saved.get("parameter.variable") == "q"
+ assert f_saved.get("metadata.shortName") == "q"
+ assert f_saved.get("vertical.level") == 600
+ assert f_saved.get("metadata.levelist") == 600
+ assert np.allclose(f_saved.values, vals_ori + 1)
+
+ # ---------------------
+ # repeated use
+ # ---------------------
+
+ f = ds_ori[0].set({
+ "values": vals_ori + 1,
+ "metadata.shortName": "q",
+ "metadata.level": 600,
+ })
+ f = f.set({
+ "values": vals_ori + 2,
+ "metadata.shortName": "pt",
+ "metadata.level": 800,
+ })
+
+ assert f.get("metadata.shortName") == "pt"
+ assert f.get("metadata.level") == 800
+ assert f.get("metadata.levelist") == 800
+ assert f.get(("metadata.date", "metadata.shortName")) == (20070101, "pt")
+ assert f.get(("metadata.shortName", "metadata.date")) == ("pt", 20070101)
+ assert np.allclose(f.values, vals_ori + 2)
+ assert np.allclose(ds_ori[0].values, vals_ori)
+
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+def test_grib_set_fieldlist_raw_metadata_and_values(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+ vals_ori = ds_ori[0].values
+
+ fields = []
+ for i in range(2):
+ f = ds_ori[i].set({
+ "values": vals_ori + i + 1,
+ "metadata.shortName": "q",
+ "metadata.level": 600,
+ })
+ fields.append(f)
+
+ ds = FieldList.from_fields(fields)
+
+ assert ds.get("parameter.variable") == ["q", "q"]
+ assert ds.get("metadata.shortName") == ["q", "q"]
+ assert ds.get("vertical.level") == [600, 600]
+ assert ds.get("metadata.level") == [600, 600]
+ assert ds.get("metadata.levelist") == [600, 600]
+ assert np.allclose(ds[0].values, vals_ori + 1)
+ assert np.allclose(ds[1].values, vals_ori + 2)
+
+ # write back to grib
+ with temp_file() as tmp:
+ ds.to_target("file", tmp)
+ ds_saved = from_source("file", tmp).to_fieldlist()
+ assert ds_saved.get("parameter.variable") == ["q", "q"]
+ assert ds_saved.get("metadata.shortName") == ["q", "q"]
+ assert ds_saved.get("vertical.level") == [600, 600]
+ assert ds_saved.get("metadata.level") == [600, 600]
+ assert ds_saved.get("metadata.levelist") == [600, 600]
+ assert np.allclose(ds_saved[0].values, vals_ori + 1)
+ assert np.allclose(ds_saved[1].values, vals_ori + 2)
+
+ # TODO: implement the following
+ # serialise
+ # pickled_f = pickle.dumps(ds)
+ # ds_1 = pickle.loads(pickled_f)
+
+ # assert ds_1.metadata("param") == ["q", "q"]
+ # assert ds_1.metadata("shortName") == ["q", "q"]
+ # assert ds_1.metadata("level") == [600, 600]
+ # assert ds_1.metadata("levelist") == [600, 600]
+
+
+@pytest.mark.parametrize("fl_type", ["file", "array", "memory"])
+# @pytest.mark.parametrize("fl_type", ["file"])
+def test_grib_set_field_mixed_metadata(fl_type):
+ ds_ori, _ = load_grib_data("test4.grib", fl_type)
+
+ f = ds_ori[0].set({
+ "metadata.shortName": "q",
+ "vertical.level": 600,
+ "labels.my_shape": (181, 360),
+ "labels.my_name": "t_500",
+ })
+
+ assert f.get("parameter.variable") == "q"
+ assert f.get("metadata.shortName") == "q"
+ assert f.get("vertical.level") == 600
+ assert f.get("metadata.levelist") == 600
+ assert f.get(("metadata.date", "parameter.variable")) == (20070101, "q")
+ assert f.get(("parameter.variable", "metadata.date")) == ("q", 20070101)
+ assert f.get("labels.my_shape") == (181, 360)
+ assert f.get("labels.my_name") == "t_500"
+
+ # write back to grib
+ with temp_file() as tmp:
+ f = ds_ori[0].set({
+ "parameter.variable": "q",
+ "vertical.level": 600,
+ })
+
+ f.to_target("file", tmp)
+ f_saved = from_source("file", tmp).to_fieldlist()[0]
+ assert f_saved.get("parameter.variable") == "q"
+ assert f_saved.get("parameter.variable") == "q"
+ assert f_saved.get("metadata.shortName") == "q"
+ assert f_saved.get("vertical.level") == 600
+ assert f_saved.get("metadata.level") == 600
+ assert f_saved.get("metadata.levelist") == 600
+ assert f_saved.get("vertical.level_type") == "pressure"
+ assert f_saved.get("metadata.typeOfLevel") == "isobaricInhPa"