diff --git a/data_pre_processing.ipynb b/data_pre_processing.ipynb
index 04fe864..0d48f0b 100644
--- a/data_pre_processing.ipynb
+++ b/data_pre_processing.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"id": "7be84f55-a429-467a-b01a-fd64cc949edb",
"metadata": {},
"outputs": [],
@@ -50,7 +50,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 43,
"id": "8aafdcf1-df25-4b8d-8ec1-6b43a186672a",
"metadata": {},
"outputs": [],
@@ -60,8 +60,8 @@
},
{
"cell_type": "code",
- "execution_count": 3,
- "id": "f4701774-c590-40cd-a550-de635d96ae07",
+ "execution_count": 15,
+ "id": "a754c167-ea55-460c-bb56-25e0c9964f3d",
"metadata": {},
"outputs": [],
"source": [
@@ -111,7 +111,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 16,
"id": "81039646-9085-4061-bbea-65bbfba3c74d",
"metadata": {},
"outputs": [],
@@ -138,7 +138,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 17,
"id": "1426df34-d486-4cfe-91a4-8d7d213c459a",
"metadata": {},
"outputs": [],
@@ -165,7 +165,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 18,
"id": "9dbbb757-c1b1-4924-a055-178506e23330",
"metadata": {},
"outputs": [],
@@ -192,7 +192,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 3,
"id": "05af817e-923b-4356-8ad3-21bc81c51ae8",
"metadata": {},
"outputs": [],
@@ -209,12 +209,63 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 6,
"id": "14d2024b-13b9-42ae-a063-ac5bc94d6cd5",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "masked_array(\n",
+ " data=[[246.4697 , 246.4697 , 246.4697 , ..., 246.4697 , 246.4697 ,\n",
+ " 246.4697 ],\n",
+ " [247.39708, 247.33827, 247.28154, ..., 247.57051, 247.51707,\n",
+ " 247.45937],\n",
+ " [248.11905, 248.00008, 247.88597, ..., 248.44518, 248.33568,\n",
+ " 248.23036],\n",
+ " ...,\n",
+ " [242.17198, 242.29324, 242.40303, ..., 241.7135 , 241.89888,\n",
+ " 242.02023],\n",
+ " [241.2053 , 241.24907, 241.28726, ..., 241.02998, 241.06673,\n",
+ " 241.13202],\n",
+ " [240.66132, 240.66132, 240.66132, ..., 240.66132, 240.66132,\n",
+ " 240.66132]],\n",
+ " mask=False,\n",
+ " fill_value=1e+20,\n",
+ " dtype=float32)"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n",
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n",
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n",
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n",
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n",
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n"
+ ]
+ }
+ ],
"source": [
- "# cube_esm1_5"
+ "cube_esm1_5"
]
},
{
@@ -493,42 +544,74 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "id": "021ad0d0-4338-4bf6-b6a0-accde93b86b4",
+ "execution_count": 4,
+ "id": "9ea9db9a-f637-4d07-967c-a2cdb900d827",
"metadata": {},
"outputs": [],
"source": [
- "## "
+ "# def calc_trend(xarr, start,end,length):\n",
+ "# #loop over years\n",
+ "# for i in np.arange(start,end,1):\n",
+ "# startyear=i\n",
+ "# endyear=i+length\n",
+ "# #extract window\n",
+ "# arr = xarr.sel(year=slice(str(startyear),str(endyear)))\n",
+ " \n",
+ "# timedim=np.arange(0,len(arr.year))\n",
+ "# arr['year'] = timedim\n",
+ "# #call polyfit to calculate trend over the window. multiply by 10 so we get units of trend per decade\n",
+ "# trnd = ((arr.polyfit(dim='year',deg=1,skipna=True)).polyfit_coefficients.isel(degree=0))*10\n",
+ " \n",
+ "# #if it's the first year, copy the trend to trnarr\n",
+ "# if i == start:\n",
+ "# trnarr = trnd\n",
+ "# #if it's the subsequent years, concatenate the trend to trnarr\n",
+ "# else:\n",
+ "# trnarr = xr.concat([trnarr,trnd],dim='year')\n",
+ "\n",
+ "# trnarr['year'] = np.arange(start,end,1)\n",
+ "\n",
+ "# return trnarr"
]
},
{
"cell_type": "code",
- "execution_count": 4,
- "id": "9ea9db9a-f637-4d07-967c-a2cdb900d827",
+ "execution_count": 7,
+ "id": "021ad0d0-4338-4bf6-b6a0-accde93b86b4",
"metadata": {},
"outputs": [],
"source": [
- "def calc_trend(xarr, start,end,length):\n",
- " #loop over years\n",
- " for i in np.arange(start,end,1):\n",
- " startyear=i\n",
- " endyear=i+length\n",
- " #extract window\n",
- " arr = xarr.sel(year=slice(str(startyear),str(endyear)))\n",
+ "## updated calc trend 6/02/26\n",
+ "def calc_trend(xarr, start, end, length):\n",
+ " \"\"\"\n",
+ " Compute running linear trends over windows of size `length` (years)\n",
+ " from `start` to `end - 1`, returning trend per decade.\n",
+ "\n",
+ " xarr must have a coordinate \"year\".\n",
+ " \"\"\"\n",
" \n",
- " timedim=np.arange(0,len(arr.year))\n",
- " arr['year'] = timedim\n",
- " #call polyfit to calculate trend over the window. multiply by 10 so we get units of trend per decade\n",
- " trnd = ((arr.polyfit(dim='year',deg=1,skipna=True)).polyfit_coefficients.isel(degree=0))*10\n",
- " \n",
- " #if it's the first year, copy the trend to trnarr\n",
- " if i == start:\n",
- " trnarr = trnd\n",
- " #if it's the subsequent years, concatenate the trend to trnarr\n",
- " else:\n",
- " trnarr = xr.concat([trnarr,trnd],dim='year')\n",
+ " trend_list = []\n",
+ "\n",
+ " for yr in range(start, end):\n",
+ " # window\n",
+ " win = xarr.sel(year=slice(str(yr), str(yr + length-1)))\n",
+ "\n",
+ " # replace year coordinate with integers 0..N-1 for polyfit\n",
+ " t = np.arange(win.year.size)\n",
+ " win = win.assign_coords(year=t)\n",
+ "\n",
+ " # polyfit: slope * 10 for trend per decade\n",
+ " tr = (win.polyfit(dim=\"year\", deg=1, skipna=True)\n",
+ " .polyfit_coefficients\n",
+ " .isel(degree=0) * 10)\n",
+ " \n",
+ " trend_list.append(tr)\n",
"\n",
- " trnarr['year'] = np.arange(start,end,1)\n",
+ " # concatenate trends along a new \"year\" dimension\n",
+ " trnarr = xr.concat(trend_list, dim=\"year\")\n",
+ "\n",
+ " # restore actual years\n",
+ " trnarr = trnarr.assign_coords(year=np.arange(start, end))\n",
"\n",
" return trnarr"
]
@@ -548,7 +631,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 8,
"id": "714ca1de-2063-4cca-9ee6-1dcb421f4ee7",
"metadata": {},
"outputs": [],
@@ -588,11 +671,22 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 9,
"id": "8565ae54-f3fa-4bfa-a16c-b10bcb55f755",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
+ "cannot be safely cast to variable data type\n",
+ " var = variable[keys]\n"
+ ]
+ }
+ ],
"source": [
+ "\n",
"pre_processing_trend_write(cube_esm1_5,\"ACCESS_ESM1_5\", output_path=\"/g/data/kj13/datasets/visualisation_projects/uncharted_future/trend\") "
]
},
@@ -648,43 +742,13 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 19,
"id": "84001a80-ec2e-4503-b725-1bd7afea0a5a",
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n",
- "/g/data/xp65/public/apps/med_conda/envs/analysis3-25.09/lib/python3.11/site-packages/iris/fileformats/netcdf/_thread_safe_nc.py:340: UserWarning: WARNING: missing_value not used since it\n",
- "cannot be safely cast to variable data type\n",
- " var = variable[keys]\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"path=\"/g/data/kj13/datasets/visualisation_projects/uncharted_future/trend\"\n",
+ "\n",
"dataset_list={\n",
" \"CESM2\":cube_cesm2,\n",
" \"NORESM2\":cube_noresm2,\n",
@@ -1486,11 +1550,1787 @@
"path=\"/g/data/kj13/datasets/visualisation_projects/uncharted_future/Rolling_SSTA/\"\n",
"pre_processing_SSTA_annually_rolling_write_ear5_monthly(cube_era5, \"ERA5-025\", output_path=path, start_year=\"1959\", end_year=\"2009\")"
]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c6bd1d3b-8cb2-4d96-8d85-bd56e34b8f3b",
+ "metadata": {},
+ "source": [
+ "### mean for multiple model trends"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "id": "5762d47b-17e7-4311-a5bf-2301553342b4",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "
<xarray.Dataset> Size: 420MB\n",
+ "Dimensions: (lon: 360, lat: 180, year: 54)\n",
+ "Coordinates:\n",
+ " * lon (lon) float64 3kB 0.5 1.5 2.5 ... 357.5 358.5 359.5\n",
+ " * lat (lat) float64 1kB -89.5 -88.5 -87.5 ... 88.5 89.5\n",
+ " * year (year) int64 432B 2015 2016 2017 ... 2066 2067 2068\n",
+ " degree int64 8B 1\n",
+ "Data variables: (12/15)\n",
+ " ACCESS-CM2ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " ACCESS-ESM1-5ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " BCC-CSM2-MRssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " CAS-ESM2-0ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " CESM2-WACCMssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(9, 180, 360), meta=np.ndarray>\n",
+ " CMCC-CM2-SR5ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " ... ...\n",
+ " MIROC6ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " MPI-ESM-1-2-HAMssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(9, 180, 360), meta=np.ndarray>\n",
+ " MRI-ESM2-0ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " NorESM2-LMssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(8, 180, 360), meta=np.ndarray>\n",
+ " NorESM2-MMssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>\n",
+ " TaiESM1ssp370 (year, lat, lon) float64 28MB dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
lon
(lon)
float64
0.5 1.5 2.5 ... 357.5 358.5 359.5
- standard_name :
- longitude
- units :
- degrees
array([ 0.5, 1.5, 2.5, ..., 357.5, 358.5, 359.5])
lat
(lat)
float64
-89.5 -88.5 -87.5 ... 88.5 89.5
- standard_name :
- latitude
- units :
- degrees
array([-89.5, -88.5, -87.5, -86.5, -85.5, -84.5, -83.5, -82.5, -81.5, -80.5,\n",
+ " -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5,\n",
+ " -69.5, -68.5, -67.5, -66.5, -65.5, -64.5, -63.5, -62.5, -61.5, -60.5,\n",
+ " -59.5, -58.5, -57.5, -56.5, -55.5, -54.5, -53.5, -52.5, -51.5, -50.5,\n",
+ " -49.5, -48.5, -47.5, -46.5, -45.5, -44.5, -43.5, -42.5, -41.5, -40.5,\n",
+ " -39.5, -38.5, -37.5, -36.5, -35.5, -34.5, -33.5, -32.5, -31.5, -30.5,\n",
+ " -29.5, -28.5, -27.5, -26.5, -25.5, -24.5, -23.5, -22.5, -21.5, -20.5,\n",
+ " -19.5, -18.5, -17.5, -16.5, -15.5, -14.5, -13.5, -12.5, -11.5, -10.5,\n",
+ " -9.5, -8.5, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5,\n",
+ " 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5,\n",
+ " 10.5, 11.5, 12.5, 13.5, 14.5, 15.5, 16.5, 17.5, 18.5, 19.5,\n",
+ " 20.5, 21.5, 22.5, 23.5, 24.5, 25.5, 26.5, 27.5, 28.5, 29.5,\n",
+ " 30.5, 31.5, 32.5, 33.5, 34.5, 35.5, 36.5, 37.5, 38.5, 39.5,\n",
+ " 40.5, 41.5, 42.5, 43.5, 44.5, 45.5, 46.5, 47.5, 48.5, 49.5,\n",
+ " 50.5, 51.5, 52.5, 53.5, 54.5, 55.5, 56.5, 57.5, 58.5, 59.5,\n",
+ " 60.5, 61.5, 62.5, 63.5, 64.5, 65.5, 66.5, 67.5, 68.5, 69.5,\n",
+ " 70.5, 71.5, 72.5, 73.5, 74.5, 75.5, 76.5, 77.5, 78.5, 79.5,\n",
+ " 80.5, 81.5, 82.5, 83.5, 84.5, 85.5, 86.5, 87.5, 88.5, 89.5])
year
(year)
int64
2015 2016 2017 ... 2066 2067 2068
array([2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026,\n",
+ " 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038,\n",
+ " 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050,\n",
+ " 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062,\n",
+ " 2063, 2064, 2065, 2066, 2067, 2068])
degree
()
int64
1
ACCESS-CM2ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
ACCESS-ESM1-5ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
BCC-CSM2-MRssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
CAS-ESM2-0ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
CESM2-WACCMssp370
(year, lat, lon)
float64
dask.array<chunksize=(9, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 4.45 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (9, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 6 chunks in 13 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
CMCC-CM2-SR5ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
CMCC-ESM2ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
CanESM5ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
FGOALS-g3ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
MIROC6ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
MPI-ESM-1-2-HAMssp370
(year, lat, lon)
float64
dask.array<chunksize=(9, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 4.45 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (9, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 6 chunks in 13 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
MRI-ESM2-0ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
NorESM2-LMssp370
(year, lat, lon)
float64
dask.array<chunksize=(8, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 3.96 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (8, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 7 chunks in 13 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
NorESM2-MMssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
TaiESM1ssp370
(year, lat, lon)
float64
dask.array<chunksize=(54, 180, 360), meta=np.ndarray>
\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | | \n",
+ " Array | \n",
+ " Chunk | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Bytes | \n",
+ " 26.70 MiB | \n",
+ " 26.70 MiB | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Shape | \n",
+ " (54, 180, 360) | \n",
+ " (54, 180, 360) | \n",
+ " \n",
+ " \n",
+ " | Dask graph | \n",
+ " 1 chunks in 2 graph layers | \n",
+ " \n",
+ " \n",
+ " | Data type | \n",
+ " float64 numpy.ndarray | \n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " \n",
+ " \n",
+ " | \n",
+ "
\n",
+ "
PandasIndex
PandasIndex(Index([ 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5,\n",
+ " ...\n",
+ " 350.5, 351.5, 352.5, 353.5, 354.5, 355.5, 356.5, 357.5, 358.5, 359.5],\n",
+ " dtype='float64', name='lon', length=360))
PandasIndex
PandasIndex(Index([-89.5, -88.5, -87.5, -86.5, -85.5, -84.5, -83.5, -82.5, -81.5, -80.5,\n",
+ " ...\n",
+ " 80.5, 81.5, 82.5, 83.5, 84.5, 85.5, 86.5, 87.5, 88.5, 89.5],\n",
+ " dtype='float64', name='lat', length=180))
PandasIndex
PandasIndex(Index([2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026,\n",
+ " 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038,\n",
+ " 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050,\n",
+ " 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062,\n",
+ " 2063, 2064, 2065, 2066, 2067, 2068],\n",
+ " dtype='int64', name='year'))
"
+ ],
+ "text/plain": [
+ " Size: 420MB\n",
+ "Dimensions: (lon: 360, lat: 180, year: 54)\n",
+ "Coordinates:\n",
+ " * lon (lon) float64 3kB 0.5 1.5 2.5 ... 357.5 358.5 359.5\n",
+ " * lat (lat) float64 1kB -89.5 -88.5 -87.5 ... 88.5 89.5\n",
+ " * year (year) int64 432B 2015 2016 2017 ... 2066 2067 2068\n",
+ " degree int64 8B 1\n",
+ "Data variables: (12/15)\n",
+ " ACCESS-CM2ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " ACCESS-ESM1-5ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " BCC-CSM2-MRssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " CAS-ESM2-0ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " CESM2-WACCMssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " CMCC-CM2-SR5ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " ... ...\n",
+ " MIROC6ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " MPI-ESM-1-2-HAMssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " MRI-ESM2-0ssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " NorESM2-LMssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " NorESM2-MMssp370 (year, lat, lon) float64 28MB dask.array\n",
+ " TaiESM1ssp370 (year, lat, lon) float64 28MB dask.array"
+ ]
+ },
+ "execution_count": 55,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import xarray as xr\n",
+ "\n",
+ "# dir_trends = '/g/data/kj13/datasets/visualisation_projects/uncharted_future/trend/models/*.nc'\n",
+ "dir_trends = '/g/data/kj13/datasets/visualisation_projects/uncharted_future/trend/models/future/*.nc'\n",
+ "\n",
+ "def _name_var(x):\n",
+ " name = x.encoding['source'].split('/')[-1]\n",
+ " name = ''.join(name.split('_')[:2])\n",
+ " x.assign_coords({'source':name})\n",
+ " return x.rename_vars({'polyfit_coefficients':name})\n",
+ "\n",
+ "mfds = xr.open_mfdataset(dir_trends, preprocess=_name_var)\n",
+ "mfds"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
+ "id": "1194718a-40a9-484f-9a8e-fce91b9df380",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mean = mfds.to_array(dim='trend').mean('trend')\n",
+ "start = mean.year[0].values\n",
+ "end = mean.year[-1].values\n",
+ "\n",
+ "output_path = '/g/data/kj13/datasets/visualisation_projects/uncharted_future/trend'\n",
+ "mean.to_netcdf(f\"{output_path}/future_cmip6_model_ts_31years_trend_{start}_{end}.nc\")"
+ ]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python [conda env:analysis3-25.09] *",
+ "display_name": "Python [conda env:analysis3-25.09]",
"language": "python",
"name": "conda-env-analysis3-25.09-py"
},
diff --git a/esmval_recipe_trends/recipe_cmip6_ts_trend.yml b/esmval_recipe_trends/recipe_cmip6_ts_trend.yml
new file mode 100644
index 0000000..b241521
--- /dev/null
+++ b/esmval_recipe_trends/recipe_cmip6_ts_trend.yml
@@ -0,0 +1,79 @@
+# ESMValTool
+#
+---
+documentation:
+ description: multi model cmip6 statistics
+ title: CMIP6 ts trends
+ authors:
+ - chun_felicity
+ maintainer:
+ - chun_felicity
+
+# datasets:
+ # - project: CMIP6
+ # exp: SSP370
+ # dataset: '*'
+ # institute: '*'
+ # ensemble: 'r*i1p1f1'
+ # grid: gn
+
+overlap: &models
+ - {dataset: ACCESS-ESM1-5, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: ACCESS-CM2, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: BCC-CSM2-MR, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: CAS-ESM2-0, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: CESM2-WACCM, project: CMIP6, ensemble: '*', grid: gn} # missing years in ensembles r2,3
+ - {dataset: CMCC-CM2-SR5, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: CMCC-ESM2, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: CanESM5, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: FGOALS-g3, project: CMIP6, ensemble: '*', grid: gn}
+ # - {dataset: IITM-ESM, project: CMIP6, ensemble: '*', grid: gn, start_year: '*', end_year: 2098} ##not in historical
+ - {dataset: MIROC6, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: MPI-ESM-1-2-HAM, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: MRI-ESM2-0, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: NorESM2-LM, project: CMIP6, ensemble: '*', grid: gn} # missing years in ensembles r2,3
+ - {dataset: NorESM2-MM, project: CMIP6, ensemble: '*', grid: gn}
+ - {dataset: TaiESM1, project: CMIP6, ensemble: '*', grid: gn}
+
+
+preprocessors:
+ all_preprocessor:
+ regrid:
+ target_grid: 1x1
+ scheme: linear
+ ensemble_statistics:
+ statistics: [mean]
+
+
+diagnostics:
+ diagnostic_metrics:
+ description: run ts in groups future, historical and obs
+ variables:
+ ts_future:
+ short_name: ts
+ mip: Amon
+ activity: ScenarioMIP
+ exp: ssp370
+ timerange: '*'
+ preprocessor: all_preprocessor
+ additional_datasets: *models
+
+ ts_obs:
+ short_name: ts
+ mip: Amon
+ timerange: '*'
+ additional_datasets:
+ - {dataset: HadISST, project: OBS, type: reanaly, tier: 2, version: 1}
+
+ ts:
+ mip: Amon
+ exp: historical
+ timerange: '*'
+ preprocessor: all_preprocessor
+ additional_datasets: *models
+
+ scripts:
+ trends:
+ script: trends.py #update filepath or run in folder
+
+
diff --git a/esmval_recipe_trends/trends.py b/esmval_recipe_trends/trends.py
new file mode 100755
index 0000000..89eeb4c
--- /dev/null
+++ b/esmval_recipe_trends/trends.py
@@ -0,0 +1,91 @@
+"""diagnostic script to plot"""
+
+import iris
+import os
+import logging
+import numpy as np
+import xarray as xr
+
+from esmvaltool.diag_scripts.shared import (
+ run_diagnostic,
+ save_figure,
+ group_metadata,
+ select_metadata,
+)
+from esmvalcore.preprocessor import (
+ climate_statistics,
+ regrid,
+ concatenate
+)
+
+
+# This part sends debug statements to stdout
+logger = logging.getLogger(os.path.basename(__file__))
+
+
+def calc_trend(xarr, start, end, length):
+ """
+ Compute running linear trends over windows of size `length` (years)
+ from `start` to `end - 1`, returning trend per decade.
+
+ xarr must have a coordinate "year".
+ """
+
+ trend_list = []
+
+ for yr in range(start, end):
+ # window
+ win = xarr.sel(year=slice(str(yr), str(yr + length-1)))
+
+ # replace year coordinate with integers 0..N-1 for polyfit
+ t = np.arange(win.year.size)
+ win = win.assign_coords(year=t)
+
+ # polyfit: slope * 10 for trend per decade
+ tr = (win.polyfit(dim="year", deg=1, skipna=True)
+ .polyfit_coefficients
+ .isel(degree=0) * 10)
+
+ trend_list.append(tr)
+
+ # concatenate trends along a new "year" dimension
+ trnarr = xr.concat(trend_list, dim="year")
+
+ # restore actual years
+ trnarr = trnarr.assign_coords(year=np.arange(start, end))
+
+ return trnarr
+
+
+def main(cfg):
+ """run on all datasets"""
+
+ input_data = cfg["input_data"].values()
+ ## group by dataset to concatenate future to historical after ensemble means
+ # for dataset, attributes in group_metadata(input_data,"dataset", sort='exp').items():
+ # in_files = [d['filename'] for d in attributes]
+ # cubes = iris.load(in_files)
+ # start_year = min([d['start_year'] for d in attributes])
+ # end_year = max([d['end_year'] for d in attributes])
+
+ # logger.info(f"Loaded {len(cubes)} cubes for {dataset}, {start_year} - {end_year}")
+ # iris.util.equalise_attributes(cubes)
+ # cube = concatenate(cubes)
+
+ ## keep future variable separate
+ for dataset in input_data:
+ input_file = dataset['filename']
+ name = dataset['dataset'] + '_' + dataset['exp']
+ cube = iris.load_cube(input_file)
+ start_year, end_year = dataset['start_year'], dataset['end_year']
+ ##
+
+ data = xr.DataArray.from_iris(cube)
+ ds_model_ann = data.groupby('time.year').mean('time')
+ ds_model_ann_trend_31 = calc_trend(ds_model_ann, start_year, end_year-31, 31)
+
+ ds_model_ann_trend_31.to_netcdf(os.path.join(cfg["work_dir"], f"{dataset}_trend_31yr{start_year}_{end_year-31}.nc"))
+
+if __name__ == "__main__":
+ with run_diagnostic() as config:
+ main(config)