From 74919067285b6acf0a6787327a303abdbd4ac846 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 21 Oct 2019 18:50:48 +0100 Subject: [PATCH 01/28] Finish implementing test_against_collect Also removes dependency on saved binary files. --- xbout/tests/test_against_collect.py | 106 ++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 20 deletions(-) diff --git a/xbout/tests/test_against_collect.py b/xbout/tests/test_against_collect.py index 45a932e1..9c69c72b 100644 --- a/xbout/tests/test_against_collect.py +++ b/xbout/tests/test_against_collect.py @@ -2,45 +2,111 @@ import numpy.testing as npt -from xbout.load import _auto_open_mfboutdataset +from xbout.load import open_boutdataset +from .test_load import create_bout_ds, create_bout_ds_list, METADATA_VARS +from boutdata import collect class TestAccuracyAgainstOldCollect: - @pytest.mark.skip - def test_single_file(self): - from boutdata import collect + def test_single_file(self, tmpdir_factory): + + # Create temp directory for files + test_dir = tmpdir_factory.mktemp("test_data") + + # Generate some test data + generated_ds = create_bout_ds(syn_data_type="linear") + generated_ds.to_netcdf(str(test_dir.join("BOUT.dmp.0.nc"))) + var = 'n' - expected = collect(var, path='./tests/data/dump_files/single', - prefix='equilibrium', xguards=False) + expected = collect(var, path=test_dir, xguards=True, yguards=False) - ds, metadata = _auto_open_mfboutdataset('./tests/data/dump_files/single/equilibrium.nc') - print(ds) + ds = open_boutdataset(test_dir.join("BOUT.dmp.0.nc")) actual = ds[var].values assert expected.shape == actual.shape npt.assert_equal(actual, expected) - @pytest.mark.skip - def test_multiple_files_along_x(self): - from boutdata import collect + + def test_multiple_files_along_x(self, tmpdir_factory): + + # Create temp directory for files + test_dir = tmpdir_factory.mktemp("test_data") + + # Generate some test data + ds_list, file_list = create_bout_ds_list("BOUT.dmp", nxpe=3, nype=1, + syn_data_type="linear") + for temp_ds, file_name in zip(ds_list, file_list): + temp_ds.to_netcdf(str(test_dir.join(str(file_name)))) + var = 'n' - expected = collect(var, path='./tests/data/dump_files/', - prefix='BOUT.dmp', xguards=False) + expected = collect(var, path=test_dir, + prefix='BOUT.dmp', xguards=True) - ds, metadata = _auto_open_mfboutdataset('./tests/data/dump_files/BOUT.dmp.*.nc') + ds = open_boutdataset(test_dir.join('BOUT.dmp.*.nc')) actual = ds[var].values assert expected.shape == actual.shape npt.assert_equal(actual, expected) - @pytest.mark.skip - def test_multiple_files_along_x(self): - ... + def test_multiple_files_along_y(self, tmpdir_factory): + + # Create temp directory for files + test_dir = tmpdir_factory.mktemp("test_data") + + # Generate some test data + ds_list, file_list = create_bout_ds_list("BOUT.dmp", nxpe=1, nype=3, + syn_data_type="linear") + for temp_ds, file_name in zip(ds_list, file_list): + temp_ds.to_netcdf(str(test_dir.join(str(file_name)))) + + var = 'n' + expected = collect(var, path=test_dir, + prefix='BOUT.dmp', xguards=True) + + ds = open_boutdataset(test_dir.join('BOUT.dmp.*.nc')) + actual = ds[var].values + + assert expected.shape == actual.shape + npt.assert_equal(actual, expected) + + + def test_multiple_files_along_xy(self, tmpdir_factory): + + # Create temp directory for files + test_dir = tmpdir_factory.mktemp("test_data") + + # Generate some test data + ds_list, file_list = create_bout_ds_list("BOUT.dmp", nxpe=3, nype=3, + syn_data_type="linear") + for temp_ds, file_name in zip(ds_list, file_list): + temp_ds.to_netcdf(str(test_dir.join(str(file_name)))) + + var = 'n' + expected = collect(var, path=test_dir, + prefix='BOUT.dmp', xguards=True) + + ds = open_boutdataset(test_dir.join('BOUT.dmp.*.nc')) + actual = ds[var].values + + assert expected.shape == actual.shape + npt.assert_equal(actual, expected) + + + def test_metadata(self, tmpdir_factory): + # Create temp directory for files + test_dir = tmpdir_factory.mktemp("test_data") + + # Generate some test data + generated_ds = create_bout_ds(syn_data_type="linear") + generated_ds.to_netcdf(str(test_dir.join("BOUT.dmp.0.nc"))) + + ds = open_boutdataset(test_dir.join('BOUT.dmp.*.nc')) - @pytest.mark.skip - def test_metadata(self): - ... + for v in METADATA_VARS: + expected = collect(v, path=test_dir) + actual = ds.bout.metadata[v] + npt.assert_equal(actual, expected) @pytest.mark.skip From 896459b2807ea04fa78d482a9db4235e1f95ab9d Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 14:26:28 +0100 Subject: [PATCH 02/28] pip-install boutdata for Travis tests Required for test_against_collect --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 641e3c31..c5fbf0b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: - "3.6" - "3.7" install: - - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov + - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata - pip install -r requirements.txt - pip install -e . script: From d7b59716d31b295e85ad9ef621494283d2230320 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 18:55:26 +0100 Subject: [PATCH 03/28] Include BOUT_VERSION in create_bout_ds This is needed for BOUT++'s 'collect' function to work correctly, as for BOUT_VERSION<3.5 it was necessary to ignore the last point in the z-grid, so collect needs to be able to check the BOUT_VERSION. --- xbout/tests/test_load.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index ad8b2d48..0c90a191 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -278,6 +278,10 @@ def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nyp n = DataArray(data, dims=['t', 'x', 'y', 'z']) ds = Dataset({'n': n, 'T': T}) + # BOUT_VERSION needed so that we know that number of points in z is MZ, not MZ-1 (as + # it was in BOUT++ before v4.0 + ds['BOUT_VERSION'] = 4.3 + # Include grid data ds['NXPE'] = nxpe ds['NYPE'] = nype From ba95d92739950d112689ee45a1d5fd0b3f9318a8 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 18:57:25 +0100 Subject: [PATCH 04/28] Remove MYPE, PE_XIND and PE_YIND from METADATA_VARS These scalar variables change from processor to processor, so are dropped when loading a dataset and cannot be checked. --- xbout/tests/test_load.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 0c90a191..42ebf408 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -342,10 +342,12 @@ def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nyp return ds -METADATA_VARS = ['NXPE', 'NYPE', 'NZPE', 'PE_XIND', 'PE_YIND', 'MYPE', 'MXG', 'MYG', 'nx', - 'ny', 'nz', 'MZ', 'MXSUB', 'MYSUB', 'MZSUB', 'ixseps1', 'ixseps2', - 'jyseps1_1', 'jyseps1_2', 'jyseps2_1', 'jyseps2_2', 'ny_inner', - 'zperiod', 'ZMIN', 'ZMAX', 'dz', 'iteration'] +# Note, MYPE, PE_XIND and PE_YIND not included, since they are different for each +# processor and so are dropped when loading datasets. +METADATA_VARS = ['BOUT_VERSION', 'NXPE', 'NYPE', 'NZPE', 'MXG', 'MYG', 'nx', 'ny', 'nz', + 'MZ', 'MXSUB', 'MYSUB', 'MZSUB', 'ixseps1', 'ixseps2', 'jyseps1_1', + 'jyseps1_2', 'jyseps2_1', 'jyseps2_2', 'ny_inner', 'zperiod', 'ZMIN', + 'ZMAX', 'dz', 'iteration'] class TestStripMetadata(): From 1c97746e5b02fc22175a87a53a8b68bddaf9e9f8 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 19:03:19 +0100 Subject: [PATCH 05/28] Drop _BOUT_PER_PROC_VARIABLES in test_load In some tests, these need to be dropped in addition to METADATA_VARS before comparing generated and loaded datasets. --- xbout/tests/test_load.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 42ebf408..fea01624 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -13,7 +13,7 @@ from xbout.load import (_check_filetype, _expand_wildcards, _expand_filepaths, _arrange_for_concatenation, _trim, _infer_contains_boundaries, - open_boutdataset) + open_boutdataset, _BOUT_PER_PROC_VARIABLES) from xbout.utils import _separate_metadata @@ -358,7 +358,8 @@ def test_strip_metadata(self): ds, metadata = _separate_metadata(original) - assert original.drop(METADATA_VARS).equals(ds) + assert original.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, + errors='ignore').equals(ds) assert metadata['NXPE'] == 1 @@ -368,7 +369,9 @@ def test_single_file(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=1, nype=1, nt=1) actual = open_boutdataset(datapath=path, keep_xboundaries=False) expected = create_bout_ds() - xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) + xrt.assert_equal(actual.load(), + expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, + errors='ignore')) def test_combine_along_x(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=4, nype=1, nt=1, @@ -378,7 +381,9 @@ def test_combine_along_x(self, tmpdir_factory, bout_xyt_example_files): bout_ds = create_bout_ds expected = concat([bout_ds(0), bout_ds(1), bout_ds(2), bout_ds(3)], dim='x', data_vars='minimal') - xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) + xrt.assert_equal(actual.load(), + expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, + errors='ignore')) def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=1, nype=3, nt=1, @@ -388,7 +393,9 @@ def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): bout_ds = create_bout_ds expected = concat([bout_ds(0), bout_ds(1), bout_ds(2)], dim='y', data_vars='minimal') - xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) + xrt.assert_equal(actual.load(), + expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, + errors='ignore')) @pytest.mark.skip def test_combine_along_t(self): @@ -408,7 +415,9 @@ def test_combine_along_xy(self, tmpdir_factory, bout_xyt_example_files): data_vars='minimal') expected = concat([line1, line2, line3], dim='y', data_vars='minimal') - xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) + xrt.assert_equal(actual.load(), + expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, + errors='ignore')) @pytest.mark.skip def test_combine_along_tx(self): From 5fdf62b49baccd032b023e014dae847e585dedde Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 18:51:40 +0100 Subject: [PATCH 06/28] Fix load._get_limit() for case when number of guard cells is 0 Returning 0 for an upper limit causes a bug, as the data is sliced with slice(lower, upper), where upper=-, but -0=0 so the slice would remove all the data. --- xbout/load.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xbout/load.py b/xbout/load.py index 8017a378..cd15e7e2 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -393,6 +393,9 @@ def _get_limit(side, dim, keep_boundaries, boundaries, guards): else: limit = None + if limit == 0: + # 0 would give incorrect result as an upper limit + limit = None return limit From d3278e59f76f04448e1228475d6cb6486b5fe5a4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 21 Oct 2019 23:23:26 +0100 Subject: [PATCH 07/28] Define custom xBOUTWarning Can then ignore it safely in the test suite. --- pytest.ini | 3 +++ xbout/load.py | 8 +++++--- xbout/tests/test_grid.py | 5 +++-- xbout/utils.py | 2 +- xbout/warning.py | 2 ++ 5 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 pytest.ini create mode 100644 xbout/warning.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..534e0fe5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::xbout.warning.xBOUTWarning diff --git a/xbout/load.py b/xbout/load.py index cd15e7e2..d0cc4881 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -9,6 +9,7 @@ from . import geometries from .utils import _set_attrs_on_all_vars, _separate_metadata, _check_filetype +from .warning import xBOUTWarning _BOUT_PER_PROC_VARIABLES = ['wall_time', 'wtime', 'wtime_rhs', 'wtime_invert', @@ -129,7 +130,8 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, ds = geometries.apply_geometry(ds, geometry) else: if info: - warn("No geometry type found, no coordinates will be added") + warn("No geometry type found, no coordinates will be added", + category=xBOUTWarning) # TODO read and store git commit hashes from output files @@ -199,7 +201,7 @@ def _expand_filepaths(datapath): " `file_cache_maxsize` global option to {} to accommodate this. " "Recommend using `xr.set_options(file_cache_maxsize=NUM)`" " to explicitly set this to a large enough value." - .format(str(len(filepaths))), UserWarning) + .format(str(len(filepaths))), category=xBOUTWarning) xr.set_options(file_cache_maxsize=len(filepaths)) return filepaths, filetype @@ -419,7 +421,7 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries): warn( "Will drop all variables containing the dimensions {} because " "they are not recognised".format(str(unrecognised_dims)[1:-1]), - UserWarning) + xBOUTWarning) grid = grid.drop_dims(unrecognised_dims) if not keep_xboundaries: diff --git a/xbout/tests/test_grid.py b/xbout/tests/test_grid.py index 4bf37157..3f6a01b4 100644 --- a/xbout/tests/test_grid.py +++ b/xbout/tests/test_grid.py @@ -7,6 +7,7 @@ from xbout.load import open_boutdataset from xbout.geometries import register_geometry, REGISTERED_GEOMETRIES +from xbout.warning import xBOUTWarning @pytest.fixture @@ -51,8 +52,8 @@ def test_open_grid_extra_dims(self, create_example_grid_file): merge([example_grid, new_var]).to_netcdf(dodgy_grid_path, engine='netcdf4') - with pytest.warns(UserWarning, match="drop all variables containing " - "the dimensions 'w'"): + with pytest.warns(xBOUTWarning, match="drop all variables containing " + "the dimensions 'w'"): result = open_boutdataset(datapath=dodgy_grid_path) assert_equal(result, example_grid) result.close() diff --git a/xbout/utils.py b/xbout/utils.py index 812b62da..727bd820 100644 --- a/xbout/utils.py +++ b/xbout/utils.py @@ -32,7 +32,7 @@ def _separate_metadata(ds): if not any(dim in ['t', 'x', 'y', 'z'] for dim in ds[var].dims)] # Save metadata as a dictionary - metadata_vals = [np.asscalar(ds[var].values) for var in scalar_vars] + metadata_vals = [ds[var].values.item() for var in scalar_vars] metadata = dict(zip(scalar_vars, metadata_vals)) return ds.drop(scalar_vars), metadata diff --git a/xbout/warning.py b/xbout/warning.py new file mode 100644 index 00000000..338f0857 --- /dev/null +++ b/xbout/warning.py @@ -0,0 +1,2 @@ +class xBOUTWarning(Warning): + pass From 93f6f91ab0e504a443eac3aa38af7e7760eed61c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 12:43:25 +0100 Subject: [PATCH 08/28] Suppress warning from xarray Many tests produce a warning from xarray like: RuntimeWarning: deallocating CachingFileManager(, '/tmp/pytest-of-***/', mode='r', kwargs={'clobber': True, 'diskless': False, 'persist': False, 'format': 'NETCDF4'}), but file is not already closed. This may indicate a bug. Ignore these warnings, as they do not tell us anything useful about xBOUT. --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 534e0fe5..e745c51d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] filterwarnings = ignore::xbout.warning.xBOUTWarning + ignore:deallocating CachingFileManager.*, but file is not already closed. This may indicate a bug\.:RuntimeWarning From 740c8227d465e972df307f4c7049824be9bd922c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 22:35:36 +0100 Subject: [PATCH 09/28] pip-install vtk in .travis.yml, workaround for Python-3.6 install In Python-3.6 the mayavi install fails if vtk is not available. mayavi is a dependency of boutdata, needed for test_against_collect. As a workaround, pip-install vtk before pip-installing boutdata. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c5fbf0b0..7fffb52e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "3.6" - "3.7" install: + - pip install --upgrade vtk # fixes install of mayavi for Python-3.6, which is a dependency of boutdata - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata - pip install -r requirements.txt - pip install -e . From 212f9137c9f4e37a588b35bd75b1eefb4c6a6fc5 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 23:15:25 +0100 Subject: [PATCH 10/28] PEP8 fixes --- xbout/tests/test_against_collect.py | 4 ---- xbout/tests/test_animate.py | 2 +- xbout/tests/test_load.py | 20 ++++++++++---------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/xbout/tests/test_against_collect.py b/xbout/tests/test_against_collect.py index 9c69c72b..909800b0 100644 --- a/xbout/tests/test_against_collect.py +++ b/xbout/tests/test_against_collect.py @@ -26,7 +26,6 @@ def test_single_file(self, tmpdir_factory): assert expected.shape == actual.shape npt.assert_equal(actual, expected) - def test_multiple_files_along_x(self, tmpdir_factory): # Create temp directory for files @@ -48,7 +47,6 @@ def test_multiple_files_along_x(self, tmpdir_factory): assert expected.shape == actual.shape npt.assert_equal(actual, expected) - def test_multiple_files_along_y(self, tmpdir_factory): # Create temp directory for files @@ -70,7 +68,6 @@ def test_multiple_files_along_y(self, tmpdir_factory): assert expected.shape == actual.shape npt.assert_equal(actual, expected) - def test_multiple_files_along_xy(self, tmpdir_factory): # Create temp directory for files @@ -92,7 +89,6 @@ def test_multiple_files_along_xy(self, tmpdir_factory): assert expected.shape == actual.shape npt.assert_equal(actual, expected) - def test_metadata(self, tmpdir_factory): # Create temp directory for files test_dir = tmpdir_factory.mktemp("test_data") diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index fb67e4be..6a6a6f2e 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -19,7 +19,7 @@ def create_test_file(tmpdir_factory): for ds, file_name in zip(ds_list, file_list): ds.to_netcdf(str(save_dir.join(str(file_name)))) - ds = open_boutdataset(save_dir.join("BOUT.dmp.*.nc")) # Open test data + ds = open_boutdataset(save_dir.join("BOUT.dmp.*.nc")) # Open test data return save_dir, ds diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index fea01624..40709f67 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -12,8 +12,8 @@ from natsort import natsorted from xbout.load import (_check_filetype, _expand_wildcards, _expand_filepaths, - _arrange_for_concatenation, _trim, _infer_contains_boundaries, - open_boutdataset, _BOUT_PER_PROC_VARIABLES) + _arrange_for_concatenation, _trim, _infer_contains_boundaries, + open_boutdataset, _BOUT_PER_PROC_VARIABLES) from xbout.utils import _separate_metadata @@ -168,7 +168,7 @@ def bout_xyt_example_files(tmpdir_factory): return _bout_xyt_example_files -def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6,2,4,7), +def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6, 2, 4, 7), nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random'): """ Mocks up a set of BOUT-like netCDF files, and return the temporary test directory containing them. @@ -197,7 +197,7 @@ def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6,2,4,7) return glob_pattern -def create_bout_ds_list(prefix, lengths=(6,2,4,7), nxpe=4, nype=2, nt=1, guards={}, +def create_bout_ds_list(prefix, lengths=(6, 2, 4, 7), nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random'): """ Mocks up a set of BOUT-like datasets. @@ -228,7 +228,7 @@ def create_bout_ds_list(prefix, lengths=(6,2,4,7), nxpe=4, nype=2, nt=1, guards= return ds_list_sorted, file_list_sorted -def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nype=1, +def create_bout_ds(syn_data_type='random', lengths=(6, 2, 4, 7), num=0, nxpe=1, nype=1, xproc=0, yproc=0, guards={}): # Set the shape of the data in this dataset @@ -247,7 +247,7 @@ def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nyp # Fill with some kind of synthetic data if syn_data_type is 'random': # Each dataset contains unique random noise - np.random.seed(seed = num) + np.random.seed(seed=num) data = np.random.randn(*shape) elif syn_data_type is 'linear': # Variables increase linearly across entire domain @@ -371,7 +371,7 @@ def test_single_file(self, tmpdir_factory, bout_xyt_example_files): expected = create_bout_ds() xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, - errors='ignore')) + errors='ignore')) def test_combine_along_x(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=4, nype=1, nt=1, @@ -383,7 +383,7 @@ def test_combine_along_x(self, tmpdir_factory, bout_xyt_example_files): data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, - errors='ignore')) + errors='ignore')) def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=1, nype=3, nt=1, @@ -395,7 +395,7 @@ def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, - errors='ignore')) + errors='ignore')) @pytest.mark.skip def test_combine_along_t(self): @@ -417,7 +417,7 @@ def test_combine_along_xy(self, tmpdir_factory, bout_xyt_example_files): data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, - errors='ignore')) + errors='ignore')) @pytest.mark.skip def test_combine_along_tx(self): From 807c8e3d0e10f335355ccfdc39e4765691cca201 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 24 Oct 2019 18:21:51 +0100 Subject: [PATCH 11/28] Add tests opening 'toroidal' and 's-alpha' geometries --- xbout/tests/test_load.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 40709f67..a67c430c 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -364,7 +364,7 @@ def test_strip_metadata(self): # TODO also test loading multiple files which have guard cells -class TestCombineNoTrim: +class TestOpen: def test_single_file(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=1, nype=1, nt=1) actual = open_boutdataset(datapath=path, keep_xboundaries=False) @@ -419,6 +419,16 @@ def test_combine_along_xy(self, tmpdir_factory, bout_xyt_example_files): expected.drop(METADATA_VARS + _BOUT_PER_PROC_VARIABLES, errors='ignore')) + def test_toroidal(self, tmpdir_factory, bout_xyt_example_files): + path = bout_xyt_example_files(tmpdir_factory, nxpe=3, nype=3, nt=1, + syn_data_type='stepped') + actual = open_boutdataset(datapath=path, geometry='toroidal') + + def test_salpha(self, tmpdir_factory, bout_xyt_example_files): + path = bout_xyt_example_files(tmpdir_factory, nxpe=3, nype=3, nt=1, + syn_data_type='stepped') + actual = open_boutdataset(datapath=path, geometry='s-alpha') + @pytest.mark.skip def test_combine_along_tx(self): ... From f8fa440fd10685bc2c72ec885476282dfb7de8ed Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 24 Oct 2019 19:06:23 +0100 Subject: [PATCH 12/28] Add option to pass grid file May be needed to provide some variables that are not saved by default to BOUT++'s output files, e.g. psixy, Rxy, Zxy. --- xbout/geometries.py | 10 ++++++++++ xbout/load.py | 16 +++++++++++++++- xbout/tests/test_load.py | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/xbout/geometries.py b/xbout/geometries.py index 3ec2c809..c22bde0f 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -108,6 +108,16 @@ def add_toroidal_geometry_coords(ds, coordinates=None): "Use the 'coordinates' argument of open_boutdataset to provide " "alternative names".format(bad_names)) + # Get extra geometry information from grid file if it's not in the dump files + needed_variables = ['psixy', 'Rxy', 'Zxy'] + for v in needed_variables: + if v not in ds: + if ds._grid is None: + raise ValueError("Grid file is required to provide %s. Pass the grid " + "file name as the 'gridfilepath' argument to " + "open_boutdataset().") + ds[v] = ds._grid[v] + # Change names of dimensions to Orthogonal Toroidal ones ds = ds.rename(y=coordinates['y']) diff --git a/xbout/load.py b/xbout/load.py index d0cc4881..923b225c 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -39,7 +39,7 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, - geometry=None, chunks={}, + geometry=None, gridfilepath=None, chunks={}, keep_xboundaries=True, keep_yboundaries=False, run_name=None, info=True): """ @@ -68,6 +68,9 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, To define a new type of geometry you need to use the `register_geometry` decorator. You are encouraged to do this for your own BOUT++ physics module, to apply relevant normalisations. + gridfilepath : str, optional + The path to a grid file, containing any variables needed to apply the geometry + specified by the 'geometry' option, which are not contained in the dump files. keep_xboundaries : bool, optional If true, keep x-direction boundary cells (the cells past the physical edges of the grid, where boundary conditions are set); increases the @@ -126,6 +129,17 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, if geometry: if info: print("Applying {} geometry conventions".format(geometry)) + + if gridfilepath is not None: + print('here in load', ds.attrs) + ds.attrs["_grid"] = open_boutdataset(gridfilepath, chunks=chunks, + keep_xboundaries=keep_xboundaries, + keep_yboundaries=keep_yboundaries, + info=info) + print('after in load', ds.attrs) + else: + ds.attrs["_grid"] = None + # Update coordinates to match particular geometry of grid ds = geometries.apply_geometry(ds, geometry) else: diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index a67c430c..0a60c3ac 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -169,7 +169,8 @@ def bout_xyt_example_files(tmpdir_factory): def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6, 2, 4, 7), - nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random'): + nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random', + grid=None): """ Mocks up a set of BOUT-like netCDF files, and return the temporary test directory containing them. @@ -184,6 +185,13 @@ def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6, 2, 4, for ds, file_name in zip(ds_list, file_list): ds.to_netcdf(str(save_dir.join(str(file_name)))) + if grid is not None: + xsize = lengths[1]*nxpe + ysize = lengths[2]*nype + grid_ds = create_bout_grid_ds(xsize=xsize, ysize=ysize, guards=guards) + print('check grid_ds',xsize,ysize,grid_ds) + grid_ds.to_netcdf(str(save_dir.join(grid + ".nc"))) + # Return a glob-like path to all files created, which has all file numbers replaced with a single asterix path = str(save_dir.join(str(file_list[-1]))) @@ -341,6 +349,21 @@ def create_bout_ds(syn_data_type='random', lengths=(6, 2, 4, 7), num=0, nxpe=1, return ds +def create_bout_grid_ds(xsize=2, ysize=4, guards={}): + + # Set the shape of the data in this dataset + mxg = guards.get('x', 0) + myg = guards.get('y', 0) + xsize += 2*mxg + ysize += 2*myg + shape = (xsize, ysize) + + data = DataArray(np.ones(shape), dims=['x', 'y']) + + ds = Dataset({'psixy': data, 'Rxy': data, 'Zxy': data, 'hthe': data}) + + return ds + # Note, MYPE, PE_XIND and PE_YIND not included, since they are different for each # processor and so are dropped when loading datasets. @@ -421,13 +444,15 @@ def test_combine_along_xy(self, tmpdir_factory, bout_xyt_example_files): def test_toroidal(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=3, nype=3, nt=1, - syn_data_type='stepped') - actual = open_boutdataset(datapath=path, geometry='toroidal') + syn_data_type='stepped', grid='grid') + actual = open_boutdataset(datapath=path, geometry='toroidal', + gridfilepath=Path(path).parent.joinpath('grid.nc')) def test_salpha(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=3, nype=3, nt=1, - syn_data_type='stepped') - actual = open_boutdataset(datapath=path, geometry='s-alpha') + syn_data_type='stepped', grid='grid') + actual = open_boutdataset(datapath=path, geometry='s-alpha', + gridfilepath=Path(path).parent.joinpath('grid.nc')) @pytest.mark.skip def test_combine_along_tx(self): From 4fad704de1f82167bc25913c97e0a96223565671 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 24 Oct 2019 22:45:55 +0100 Subject: [PATCH 13/28] Add 'hthe' from grid in s-alpha geometry Need to add hthe before getting toroidal coordinates, as dimension names are changed only in the Dataset, not in the _grid member variable. 'r' coordinate is created as 1d, so selecting 'theta=0' part is an error. --- xbout/geometries.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/xbout/geometries.py b/xbout/geometries.py index c22bde0f..86deca49 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -155,6 +155,14 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): coordinates = _set_default_toroidal_coordinates(coordinates) + # Add 'hthe' from grid file, needed below for radial coordinate + if not 'hthe' in ds: + if ds._grid is None: + raise ValueError("Grid file is required to provide %s. Pass the grid " + "file name as the 'gridfilepath' argument to " + "open_boutdataset().") + ds['hthe'] = ds._grid['hthe'] + ds = add_toroidal_geometry_coords(ds, coordinates=coordinates) # Add 1D radial coordinate @@ -166,7 +174,4 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): ds = ds.set_coords('r') ds = ds.rename(x='r') - # Simplify psi to be radially-varying only - ds['r'] = ds['r'].isel({coordinates['y']: 0}).squeeze(drop=True) - return ds From 181a691807f2ed1031f60dd64a332cdce19add04 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 25 Oct 2019 09:08:15 +0100 Subject: [PATCH 14/28] Travis workaround to skip xarray-0.14.0, which breaks the tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7fffb52e..ba5fefca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "3.7" install: - pip install --upgrade vtk # fixes install of mayavi for Python-3.6, which is a dependency of boutdata - - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata + - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata "xarray!=0.14.0" - pip install -r requirements.txt - pip install -e . script: From 60b9d9a94da1450192fe18bc181df3cb097be53a Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 25 Oct 2019 18:27:55 +0100 Subject: [PATCH 15/28] Fix PEP8 issuses, remove debugging print statements --- xbout/geometries.py | 2 +- xbout/load.py | 2 -- xbout/tests/test_load.py | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/xbout/geometries.py b/xbout/geometries.py index 86deca49..db6146c6 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -156,7 +156,7 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): coordinates = _set_default_toroidal_coordinates(coordinates) # Add 'hthe' from grid file, needed below for radial coordinate - if not 'hthe' in ds: + if 'hthe' not in ds: if ds._grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " diff --git a/xbout/load.py b/xbout/load.py index 923b225c..10c8dc45 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -131,12 +131,10 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, print("Applying {} geometry conventions".format(geometry)) if gridfilepath is not None: - print('here in load', ds.attrs) ds.attrs["_grid"] = open_boutdataset(gridfilepath, chunks=chunks, keep_xboundaries=keep_xboundaries, keep_yboundaries=keep_yboundaries, info=info) - print('after in load', ds.attrs) else: ds.attrs["_grid"] = None diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 0a60c3ac..c7dbba2d 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -76,7 +76,6 @@ def test_no_files(self, tmpdir): with pytest.raises(IOError): path = Path(str(files_dir.join('run*/example.*.nc'))) actual_filepaths = _expand_filepaths(path) - print(actual_filepaths) @pytest.fixture() @@ -189,7 +188,6 @@ def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6, 2, 4, xsize = lengths[1]*nxpe ysize = lengths[2]*nype grid_ds = create_bout_grid_ds(xsize=xsize, ysize=ysize, guards=guards) - print('check grid_ds',xsize,ysize,grid_ds) grid_ds.to_netcdf(str(save_dir.join(grid + ".nc"))) # Return a glob-like path to all files created, which has all file numbers replaced with a single asterix @@ -349,6 +347,7 @@ def create_bout_ds(syn_data_type='random', lengths=(6, 2, 4, 7), num=0, nxpe=1, return ds + def create_bout_grid_ds(xsize=2, ysize=4, guards={}): # Set the shape of the data in this dataset From cfc039c32d4097e6b16cd3189b9faa68a182b4d6 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 26 Oct 2019 11:53:27 +0100 Subject: [PATCH 16/28] Don't store _grid in dataset attrs Doing this prevented the dataset being saved to netCDF file. Also use _open_grid instead of open_boutdataset to open the grid file so that the grid dataset does not have metadata added to the DataArray variables. These variables are added as coordinates and become members of the DataArrays representing simulation variables; if they have a metadata dict, it is not possible to save them to netCDF. --- xbout/boutdataset.py | 3 --- xbout/geometries.py | 8 ++++---- xbout/load.py | 9 ++++----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/xbout/boutdataset.py b/xbout/boutdataset.py index e4e47d30..364d0731 100644 --- a/xbout/boutdataset.py +++ b/xbout/boutdataset.py @@ -19,7 +19,6 @@ def __init__(self, ds): self.data = ds self.metadata = ds.attrs.get('metadata') # None if just grid file self.options = ds.attrs.get('options') # None if no inp file - self.grid = ds.attrs.get('grid') # None if no grid file def __str__(self): """ @@ -34,8 +33,6 @@ def __str__(self): "Metadata:\n{}\n".format(styled(self.metadata)) if self.options: text += "Options:\n{}".format(styled(self.options)) - if self.grid: - text += "Grid:\n{}".format(styled(self.grid)) return text #def __repr__(self): diff --git a/xbout/geometries.py b/xbout/geometries.py index db6146c6..e8c395df 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -112,11 +112,11 @@ def add_toroidal_geometry_coords(ds, coordinates=None): needed_variables = ['psixy', 'Rxy', 'Zxy'] for v in needed_variables: if v not in ds: - if ds._grid is None: + if ds.bout._grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " "open_boutdataset().") - ds[v] = ds._grid[v] + ds[v] = ds.bout._grid[v] # Change names of dimensions to Orthogonal Toroidal ones ds = ds.rename(y=coordinates['y']) @@ -157,11 +157,11 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): # Add 'hthe' from grid file, needed below for radial coordinate if 'hthe' not in ds: - if ds._grid is None: + if ds.bout._grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " "open_boutdataset().") - ds['hthe'] = ds._grid['hthe'] + ds['hthe'] = ds.bout._grid['hthe'] ds = add_toroidal_geometry_coords(ds, coordinates=coordinates) diff --git a/xbout/load.py b/xbout/load.py index 10c8dc45..61a9b353 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -131,12 +131,11 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, print("Applying {} geometry conventions".format(geometry)) if gridfilepath is not None: - ds.attrs["_grid"] = open_boutdataset(gridfilepath, chunks=chunks, - keep_xboundaries=keep_xboundaries, - keep_yboundaries=keep_yboundaries, - info=info) + ds.bout._grid = _open_grid(gridfilepath, chunks=chunks, + keep_xboundaries=keep_xboundaries, + keep_yboundaries=keep_yboundaries) else: - ds.attrs["_grid"] = None + ds.bout._grid = None # Update coordinates to match particular geometry of grid ds = geometries.apply_geometry(ds, geometry) From 7ae0f18d4621c144323426eee34dfe86724dd1a5 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 26 Oct 2019 12:17:45 +0100 Subject: [PATCH 17/28] Test saving BoutDataset with geometry --- xbout/tests/test_load.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index c7dbba2d..cb0f40fc 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -447,12 +447,20 @@ def test_toroidal(self, tmpdir_factory, bout_xyt_example_files): actual = open_boutdataset(datapath=path, geometry='toroidal', gridfilepath=Path(path).parent.joinpath('grid.nc')) + # check dataset can be saved + save_dir = tmpdir_factory.mktemp('data') + actual.bout.save(str(save_dir.join('boutdata.nc'))) + def test_salpha(self, tmpdir_factory, bout_xyt_example_files): path = bout_xyt_example_files(tmpdir_factory, nxpe=3, nype=3, nt=1, syn_data_type='stepped', grid='grid') actual = open_boutdataset(datapath=path, geometry='s-alpha', gridfilepath=Path(path).parent.joinpath('grid.nc')) + # check dataset can be saved + save_dir = tmpdir_factory.mktemp('data') + actual.bout.save(str(save_dir.join('boutdata.nc'))) + @pytest.mark.skip def test_combine_along_tx(self): ... From 3fc4f02b89ab8a07aed39a7dfee808ba3782c329 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 26 Oct 2019 12:24:23 +0100 Subject: [PATCH 18/28] Do not save hthe in dataset for s-alpha geometry ...when hthe is loaded from the grid file. Variables from the grid file do not have all the correct metadata, so cause errors when trying to save to netCDF. --- xbout/geometries.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/xbout/geometries.py b/xbout/geometries.py index e8c395df..8ee6ce61 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -157,11 +157,14 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): # Add 'hthe' from grid file, needed below for radial coordinate if 'hthe' not in ds: + hthe_from_grid = True if ds.bout._grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " "open_boutdataset().") ds['hthe'] = ds.bout._grid['hthe'] + else: + hthe_from_grid = False ds = add_toroidal_geometry_coords(ds, coordinates=coordinates) @@ -174,4 +177,8 @@ def add_s_alpha_geometry_coords(ds, coordinates=None): ds = ds.set_coords('r') ds = ds.rename(x='r') + if hthe_from_grid: + # remove hthe because it does not have correct metadata + del ds['hthe'] + return ds From 75f3072f5b93c191b5d99d963d63e1ff3e77fe02 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 26 Oct 2019 21:40:04 +0100 Subject: [PATCH 19/28] Remove workaround for mayavi dependency of boutdata Depedency has been made optional by an update to boututils. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ba5fefca..9e244f42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "3.6" - "3.7" install: - - pip install --upgrade vtk # fixes install of mayavi for Python-3.6, which is a dependency of boutdata - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata "xarray!=0.14.0" - pip install -r requirements.txt - pip install -e . From 7c0511225efaeaa6ec80941452fdca60e976dfa4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 26 Oct 2019 21:40:04 +0100 Subject: [PATCH 20/28] Remove workaround for mayavi dependency of boutdata Depedency has been made optional by an update to boututils. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7fffb52e..c5fbf0b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "3.6" - "3.7" install: - - pip install --upgrade vtk # fixes install of mayavi for Python-3.6, which is a dependency of boutdata - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata - pip install -r requirements.txt - pip install -e . From 4902a7432e550a15249ce2b4cfbccde44902a1f7 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 25 Oct 2019 09:08:15 +0100 Subject: [PATCH 21/28] Travis workaround to skip xarray-0.14.0, which breaks the tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c5fbf0b0..9e244f42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: - "3.6" - "3.7" install: - - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata + - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov boutdata "xarray!=0.14.0" - pip install -r requirements.txt - pip install -e . script: From 4a869477c128a1167cfce94ca151ee0816dd39e8 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 28 Oct 2019 21:45:03 +0000 Subject: [PATCH 22/28] Revert addition of xBOUTWarning Instead use UserWarning, and in pytest.ini set a filter on the warning message. --- pytest.ini | 2 +- xbout/load.py | 9 +++------ xbout/tests/test_grid.py | 5 ++--- xbout/warning.py | 2 -- 4 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 xbout/warning.py diff --git a/pytest.ini b/pytest.ini index e745c51d..ac2cecc2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] filterwarnings = - ignore::xbout.warning.xBOUTWarning + ignore:No geometry type found, no coordinates will be added:UserWarning ignore:deallocating CachingFileManager.*, but file is not already closed. This may indicate a bug\.:RuntimeWarning diff --git a/xbout/load.py b/xbout/load.py index d0cc4881..7b7b8db7 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -9,7 +9,6 @@ from . import geometries from .utils import _set_attrs_on_all_vars, _separate_metadata, _check_filetype -from .warning import xBOUTWarning _BOUT_PER_PROC_VARIABLES = ['wall_time', 'wtime', 'wtime_rhs', 'wtime_invert', @@ -130,8 +129,7 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, ds = geometries.apply_geometry(ds, geometry) else: if info: - warn("No geometry type found, no coordinates will be added", - category=xBOUTWarning) + warn("No geometry type found, no coordinates will be added") # TODO read and store git commit hashes from output files @@ -201,7 +199,7 @@ def _expand_filepaths(datapath): " `file_cache_maxsize` global option to {} to accommodate this. " "Recommend using `xr.set_options(file_cache_maxsize=NUM)`" " to explicitly set this to a large enough value." - .format(str(len(filepaths))), category=xBOUTWarning) + .format(str(len(filepaths)))) xr.set_options(file_cache_maxsize=len(filepaths)) return filepaths, filetype @@ -420,8 +418,7 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries): # pytest warnings capture - doesn't match strings containing brackets warn( "Will drop all variables containing the dimensions {} because " - "they are not recognised".format(str(unrecognised_dims)[1:-1]), - xBOUTWarning) + "they are not recognised".format(str(unrecognised_dims)[1:-1])) grid = grid.drop_dims(unrecognised_dims) if not keep_xboundaries: diff --git a/xbout/tests/test_grid.py b/xbout/tests/test_grid.py index 3f6a01b4..4bf37157 100644 --- a/xbout/tests/test_grid.py +++ b/xbout/tests/test_grid.py @@ -7,7 +7,6 @@ from xbout.load import open_boutdataset from xbout.geometries import register_geometry, REGISTERED_GEOMETRIES -from xbout.warning import xBOUTWarning @pytest.fixture @@ -52,8 +51,8 @@ def test_open_grid_extra_dims(self, create_example_grid_file): merge([example_grid, new_var]).to_netcdf(dodgy_grid_path, engine='netcdf4') - with pytest.warns(xBOUTWarning, match="drop all variables containing " - "the dimensions 'w'"): + with pytest.warns(UserWarning, match="drop all variables containing " + "the dimensions 'w'"): result = open_boutdataset(datapath=dodgy_grid_path) assert_equal(result, example_grid) result.close() diff --git a/xbout/warning.py b/xbout/warning.py deleted file mode 100644 index 338f0857..00000000 --- a/xbout/warning.py +++ /dev/null @@ -1,2 +0,0 @@ -class xBOUTWarning(Warning): - pass From bf9de54b741686b47b6f274cf0dc31b6393533fe Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 15:50:54 +0000 Subject: [PATCH 23/28] Don't try to store grid dataset after reading coordinates Previously, a gridfile was loaded and stored as an attribute of the Dataset and DataArrays, then it was attempted to store the grid Dataset as a member of the BoutDataset accessor (this does not work because accessors are not permanent). As of BOUT++ v4.3.0, the only information we need to load from the grid file is some coordinates (e.g. psixy, Rxy, Zxy for 'toroidal' geometry), so we can just open the grid file, set the coordinates and not save the grid. This has the advantage that DataArrays inherit the coordinates from their parent Dataset without having to do special things when loading, or causing problems when saving to netCDF. --- xbout/boutdataarray.py | 3 --- xbout/geometries.py | 27 ++++++++++++++++++--------- xbout/load.py | 6 +++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/xbout/boutdataarray.py b/xbout/boutdataarray.py index 06eff929..bbcc8d51 100644 --- a/xbout/boutdataarray.py +++ b/xbout/boutdataarray.py @@ -20,7 +20,6 @@ def __init__(self, da): self.data = da self.metadata = da.attrs.get('metadata') # None if just grid file self.options = da.attrs.get('options') # None if no inp file - self.grid = da.attrs.get('grid') # None if no grid file def __str__(self): """ @@ -35,8 +34,6 @@ def __str__(self): "Metadata:\n{}\n".format(styled(self.metadata)) if self.options: text += "Options:\n{}".format(styled(self.options)) - if self.grid: - text += "Grid:\n{}".format(styled(self.grid)) return text def animate2D(self, animate_over='t', x='x', y='y', animate=True, diff --git a/xbout/geometries.py b/xbout/geometries.py index 8ee6ce61..2e21cc82 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -16,7 +16,7 @@ class UnregisteredGeometryError(Exception): # desired -def apply_geometry(ds, geometry_name, coordinates=None): +def apply_geometry(ds, geometry_name, *, coordinates=None, grid=None): """ Parameters @@ -48,7 +48,16 @@ def apply_geometry(ds, geometry_name, coordinates=None): have been registered.""".format(geometry_name)) raise UnregisteredGeometryError(message) - updated_ds = add_geometry_coords(ds, coordinates=coordinates) + # User-registered functions may accept 'coordinates' and 'grid' arguments, but do not + # have to as long as they are not used + if coordinates is not None and grid is not None: + updated_ds = add_geometry_coords(ds, coordinates=coordinates, grid=grid) + elif coordinates is not None: + updated_ds = add_geometry_coords(ds, coordinates=coordinates) + elif grid is not None: + updated_ds = add_geometry_coords(ds, grid=grid) + else: + updated_ds = add_geometry_coords(ds) return updated_ds @@ -97,7 +106,7 @@ def _set_default_toroidal_coordinates(coordinates): @register_geometry('toroidal') -def add_toroidal_geometry_coords(ds, coordinates=None): +def add_toroidal_geometry_coords(ds, *, coordinates=None, grid=None): coordinates = _set_default_toroidal_coordinates(coordinates) @@ -112,11 +121,11 @@ def add_toroidal_geometry_coords(ds, coordinates=None): needed_variables = ['psixy', 'Rxy', 'Zxy'] for v in needed_variables: if v not in ds: - if ds.bout._grid is None: + if grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " "open_boutdataset().") - ds[v] = ds.bout._grid[v] + ds[v] = grid[v] # Change names of dimensions to Orthogonal Toroidal ones ds = ds.rename(y=coordinates['y']) @@ -151,22 +160,22 @@ def add_toroidal_geometry_coords(ds, coordinates=None): @register_geometry('s-alpha') -def add_s_alpha_geometry_coords(ds, coordinates=None): +def add_s_alpha_geometry_coords(ds, *, coordinates=None, grid=None): coordinates = _set_default_toroidal_coordinates(coordinates) # Add 'hthe' from grid file, needed below for radial coordinate if 'hthe' not in ds: hthe_from_grid = True - if ds.bout._grid is None: + if grid is None: raise ValueError("Grid file is required to provide %s. Pass the grid " "file name as the 'gridfilepath' argument to " "open_boutdataset().") - ds['hthe'] = ds.bout._grid['hthe'] + ds['hthe'] = grid['hthe'] else: hthe_from_grid = False - ds = add_toroidal_geometry_coords(ds, coordinates=coordinates) + ds = add_toroidal_geometry_coords(ds, coordinates=coordinates, grid=grid) # Add 1D radial coordinate if 'r' in ds: diff --git a/xbout/load.py b/xbout/load.py index 69b90b43..7324bff3 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -130,14 +130,14 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, print("Applying {} geometry conventions".format(geometry)) if gridfilepath is not None: - ds.bout._grid = _open_grid(gridfilepath, chunks=chunks, + grid = _open_grid(gridfilepath, chunks=chunks, keep_xboundaries=keep_xboundaries, keep_yboundaries=keep_yboundaries) else: - ds.bout._grid = None + grid = None # Update coordinates to match particular geometry of grid - ds = geometries.apply_geometry(ds, geometry) + ds = geometries.apply_geometry(ds, geometry, grid=grid) else: if info: warn("No geometry type found, no coordinates will be added") From 992218849d1321551f91755f16fad1fa5f5c54d8 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 16:29:31 +0000 Subject: [PATCH 24/28] Remove skipped grid_merge test We no longer merge grid data from a grid file into the Dataset, so this test is not needed. --- xbout/tests/test_grid.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/xbout/tests/test_grid.py b/xbout/tests/test_grid.py index 4bf37157..3e48368b 100644 --- a/xbout/tests/test_grid.py +++ b/xbout/tests/test_grid.py @@ -57,10 +57,6 @@ def test_open_grid_extra_dims(self, create_example_grid_file): assert_equal(result, example_grid) result.close() - @pytest.mark.skip - def test_open_grid_merge_ds(self): - ... - def test_open_grid_apply_geometry(self, create_example_grid_file): @register_geometry(name="Schwarzschild") def add_schwarzschild_coords(ds, coordinates=None): From d7f8163a6cea4f14246b88821450b88606c70f18 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 16:35:42 +0000 Subject: [PATCH 25/28] Fix passing MXG to _open_grid MXG is not contained in the grid file, so needs to be passed as an argument to _open_grid: read from the Dataset if one is being opened, otherwise defaults to 2. --- xbout/load.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index 7324bff3..73131a5c 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -132,7 +132,8 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, if gridfilepath is not None: grid = _open_grid(gridfilepath, chunks=chunks, keep_xboundaries=keep_xboundaries, - keep_yboundaries=keep_yboundaries) + keep_yboundaries=keep_yboundaries, + mxg=ds.metadata['MXG']) else: grid = None @@ -410,7 +411,7 @@ def _get_limit(side, dim, keep_boundaries, boundaries, guards): return limit -def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries): +def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries, mxg=2): """ Opens a single grid file. Implements slightly different logic for boundaries to deal with different conventions in a BOUT grid file. @@ -433,7 +434,7 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries): grid = grid.drop_dims(unrecognised_dims) if not keep_xboundaries: - xboundaries = int(grid.metadata['MXG']) + xboundaries = mxg if xboundaries > 0: grid = grid.isel(x=slice(xboundaries, -xboundaries, None)) if not keep_yboundaries: From 0583eac914c5ba5785627d53599737c2ea02efe3 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 20:13:35 +0000 Subject: [PATCH 26/28] Docstring for 'grid' argument to applyGeometry --- xbout/geometries.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xbout/geometries.py b/xbout/geometries.py index 3bca162f..9b701dd7 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -30,6 +30,10 @@ def apply_geometry(ds, geometry_name, *, coordinates=None, grid=None): corresponding to 'x', 'y' and 'z' keys in the passed dict are used as the names of the dimensions. Any not passed are given default values. If not specified, default names are chosen. + grid : Dataset, optional + Dataset containing extra geometrical information not stored in the dump files that + is needed to add coordinates for the geometry being applied. For example, should + contain 2d arrays Rxy, Zxy and psixy for toroidal geometry. Returns ------- From dcda0d3028537c02a3c198ef5b1a065bcb9a333c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 22:13:47 +0000 Subject: [PATCH 27/28] Skip test_against_collect if boutdata.collect is not available If boutdata.collect, 'old collect', is not available, then skip tests comparing to it. Ensures users who do not install old python tools can still run the tests. --- xbout/tests/test_against_collect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xbout/tests/test_against_collect.py b/xbout/tests/test_against_collect.py index 909800b0..58052f14 100644 --- a/xbout/tests/test_against_collect.py +++ b/xbout/tests/test_against_collect.py @@ -5,7 +5,8 @@ from xbout.load import open_boutdataset from .test_load import create_bout_ds, create_bout_ds_list, METADATA_VARS -from boutdata import collect +boutdata = pytest.importorskip("boutdata", reason="boutdata is not available") +collect = boutdata.collect class TestAccuracyAgainstOldCollect: def test_single_file(self, tmpdir_factory): From 5370f7d44ce7fdbcca0f800be42633cbe7c7e953 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Dec 2019 22:23:26 +0000 Subject: [PATCH 28/28] Fix PEP8 issues --- xbout/geometries.py | 6 +++--- xbout/load.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xbout/geometries.py b/xbout/geometries.py index 9b701dd7..a938952d 100644 --- a/xbout/geometries.py +++ b/xbout/geometries.py @@ -31,9 +31,9 @@ def apply_geometry(ds, geometry_name, *, coordinates=None, grid=None): of the dimensions. Any not passed are given default values. If not specified, default names are chosen. grid : Dataset, optional - Dataset containing extra geometrical information not stored in the dump files that - is needed to add coordinates for the geometry being applied. For example, should - contain 2d arrays Rxy, Zxy and psixy for toroidal geometry. + Dataset containing extra geometrical information not stored in the dump files + that is needed to add coordinates for the geometry being applied. For example, + should contain 2d arrays Rxy, Zxy and psixy for toroidal geometry. Returns ------- diff --git a/xbout/load.py b/xbout/load.py index 73131a5c..171b05e8 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -131,9 +131,9 @@ def open_boutdataset(datapath='./BOUT.dmp.*.nc', inputfilepath=None, if gridfilepath is not None: grid = _open_grid(gridfilepath, chunks=chunks, - keep_xboundaries=keep_xboundaries, - keep_yboundaries=keep_yboundaries, - mxg=ds.metadata['MXG']) + keep_xboundaries=keep_xboundaries, + keep_yboundaries=keep_yboundaries, + mxg=ds.metadata['MXG']) else: grid = None