From bc5f6e66f77bdd3180f22ce42761447c6c363504 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 08:55:23 -0500 Subject: [PATCH 1/8] adding CAMx support with some xarray --- src/PseudoNetCDF/coordutil.py | 24 ++++++++++++------------ src/PseudoNetCDF/core/_files.py | 18 ++++++++++++++---- src/PseudoNetCDF/core/_variables.py | 2 +- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/PseudoNetCDF/coordutil.py b/src/PseudoNetCDF/coordutil.py index 20a250f..ba37f6b 100644 --- a/src/PseudoNetCDF/coordutil.py +++ b/src/PseudoNetCDF/coordutil.py @@ -423,9 +423,9 @@ def wrapper(first, instr): """ % outdict -def getprojwkt(ifile, withgrid=False): +def getprojwkt(ifile, withgrid=False, fromorigin=False): import osr - proj4str = getproj4(ifile, withgrid=withgrid) + proj4str = getproj4(ifile, withgrid=withgrid, fromorigin=fromorigin) srs = osr.SpatialReference() # Imports WKT to Spatial Reference Object @@ -434,11 +434,11 @@ def getprojwkt(ifile, withgrid=False): return srs.ExportToWkt() -def basemap_from_file(ifile, withgrid=False, **kwds): +def basemap_from_file(ifile, withgrid=False, fromorigin=False, **kwds): """ Typically, the user will need to provide some options """ - proj4 = getproj4(ifile, withgrid=withgrid) + proj4 = getproj4(ifile, withgrid=withgrid, fromorigin=fromorigin) basemap_options = basemap_options_from_proj4(proj4, **kwds) if 'llcrnrx' in basemap_options: if 'urcrnrx' in kwds: @@ -522,7 +522,7 @@ def basemap_from_proj4(proj4, **kwds): return bmap -def getproj4_from_cf_var(gridmapping, withgrid=False): +def getproj4_from_cf_var(gridmapping, withgrid=False, fromorigin=False): mapstr_bits = OrderedDict() gname = getattr(gridmapping, 'grid_mapping_name') pv4s = dict(lambert_conformal_conic='lcc', @@ -558,9 +558,9 @@ def getproj4_from_cf_var(gridmapping, withgrid=False): mapstr_bits['lon_0'] = pv elif pk == 'latitude_of_projection_origin': mapstr_bits['lat_0'] = pv - elif pk == 'false_easting': + elif pk == 'false_easting' and not fromorigin: mapstr_bits['x_0'] = pv - elif pk == 'false_northing': + elif pk == 'false_northing' and not fromorigin: mapstr_bits['y_0'] = pv elif pk == 'scale_factor_at_projection_origin': mapstr_bits['k_0'] = pv @@ -588,9 +588,9 @@ def getproj4_from_cf_var(gridmapping, withgrid=False): return mapstr -def getproj(ifile, withgrid=False): +def getproj(ifile, withgrid=False, fromorigin=False): import pyproj - proj4str = getproj4(ifile, withgrid=withgrid) + proj4str = getproj4(ifile, withgrid=withgrid, fromorigin=fromorigin) preserve_units = withgrid # pyproj adds +units=m, which is not right for latlon/lonlat if '+proj=lonlat' in proj4str or '+proj=latlon' in proj4str: @@ -598,7 +598,7 @@ def getproj(ifile, withgrid=False): return pyproj.Proj(proj4str, preserve_units=preserve_units) -def getproj4(ifile, withgrid=False): +def getproj4(ifile, withgrid=False, fromorigin=False): """ Arguments: ifile - PseudoNetCDF file @@ -614,7 +614,7 @@ def getproj4(ifile, withgrid=False): for k in 'P_GAM P_ALP P_BET XORIG YORIG XCELL YCELL'.split()]) ): gridmapping = getmapdef(ifile, add=False) - mapstr = getproj4_from_cf_var(gridmapping, withgrid=withgrid) + mapstr = getproj4_from_cf_var(gridmapping, withgrid=withgrid, fromorigin=fromorigin) if withgrid: dx = min(ifile.XCELL, ifile.YCELL) if ifile.XCELL != ifile.YCELL: @@ -656,7 +656,7 @@ def getproj4(ifile, withgrid=False): mapstr = '+proj=lonlat' else: gridmapping = ifile.variables[gridmappings[0]] - mapstr = getproj4_from_cf_var(gridmapping, withgrid=withgrid) + mapstr = getproj4_from_cf_var(gridmapping, withgrid=withgrid, fromorigin=fromorigin) else: warn('No known grid mapping; assuming lonlat') mapstr = '+proj=lonlat' diff --git a/src/PseudoNetCDF/core/_files.py b/src/PseudoNetCDF/core/_files.py index d8e5875..86c2f23 100644 --- a/src/PseudoNetCDF/core/_files.py +++ b/src/PseudoNetCDF/core/_files.py @@ -55,6 +55,14 @@ class PseudoNetCDFFile(PseudoNetCDFSelfReg, object): methods that a file should present to act like a netCDF file using the Scientific.IO.NetCDF.NetCDFFile interface. """ + def xarray(self): + import xarray as xr + data_vars = { + k: v.xarray() for k, v in self.variables.items() + if k not in self.dimensions + } + attrs = {k: self.getncattr(k) for k in self.ncattrs()} + return xr.Dataset(data_vars, attrs=attrs) def getMap(self, maptype='basemap_auto', **kwds): """ @@ -113,7 +121,7 @@ def getMap(self, maptype='basemap_auto', **kwds): raise ValueError( 'maptype must be basemap, basemap_auto, or cartopy') - def getproj(self, withgrid=False, projformat='pyproj'): + def getproj(self, withgrid=False, projformat='pyproj', fromorigin=False): """ Description @@ -125,6 +133,8 @@ def getproj(self, withgrid=False, projformat='pyproj'): 'pyproj' (default), 'proj4' or 'wkt' allows function to return a pyproj projection object or a string in the format of proj4 or WKT + fromorigin : boolean + Ignore false easting and false northing offsets Returns ------- @@ -133,13 +143,13 @@ def getproj(self, withgrid=False, projformat='pyproj'): """ if projformat == 'pyproj': from PseudoNetCDF.coordutil import getproj - return getproj(self, withgrid=withgrid) + return getproj(self, withgrid=withgrid, fromorigin=fromorigin) elif projformat == 'proj4': from PseudoNetCDF.coordutil import getproj4 - return getproj4(self, withgrid=withgrid) + return getproj4(self, withgrid=withgrid, fromorigin=fromorigin) elif projformat == 'wkt': from PseudoNetCDF.coordutil import getprojwkt - return getprojwkt(self, withgrid=withgrid) + return getprojwkt(self, withgrid=withgrid, fromorigin=fromorigin) else: raise ValueError('projformat must be pyproj, proj4 or wkt') diff --git a/src/PseudoNetCDF/core/_variables.py b/src/PseudoNetCDF/core/_variables.py index 648fd30..138e1b9 100644 --- a/src/PseudoNetCDF/core/_variables.py +++ b/src/PseudoNetCDF/core/_variables.py @@ -67,7 +67,7 @@ def get_coord(self, coordn): coordi = list(self.dimensions).index(coordn) coordv = np.arange(self.shape[coordi]) else: - coordv = (v.dimensions, v.xarray(iscoord=True)) + coordv = (v.dimensions, v.array()) else: coordi = list(self.dimensions).index(coordn) coordv = np.arange(self.shape[coordi]) From acedbf2116d26758fd7fbf066b05921cce940f6f Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:46:19 -0500 Subject: [PATCH 2/8] being more specific on requirements --- docs/requirements.txt | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 6ec2466..51b05c5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,13 +5,14 @@ h5netcdf matplotlib pytest flake8 -numpy -pandas +numpy>=1.2,<2 +pandas<3 scipy pyyaml netcdf4 xarray pyproj +packaging sphinx sphinx-rtd-theme sphinx-gallery diff --git a/setup.py b/setup.py index 2509be5..6962f26 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ def find_data(): ] requires_list = [ - 'numpy>=1.2', 'netCDF4', 'scipy', 'matplotlib', 'pyyaml', 'pandas', + 'numpy>=1.2,<2', 'netCDF4', 'scipy', 'matplotlib', 'pyyaml', 'pandas<3', 'packaging' ] if sys.version_info.major == 3: From 15983ade4c0782f32dc501cadc1042a8e1cdc815 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:48:25 -0500 Subject: [PATCH 3/8] Removing coverage dependence After v7.4.3, coverage causes netCDF4 to report a runtime error RuntimeError: NetCDF: Not a valid ID It is not clear to me why this happens, but it is reliable. v7.4.4 and above cause the error, which can be bypassed by downgrading coverage or by using pytest directly. The test does not strictly depend on increasing coverage, so I am removing it. --- tox.ini | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index c513911..4533028 100644 --- a/tox.ini +++ b/tox.ini @@ -5,18 +5,20 @@ envlist = py{36,39,311,312} [coverage:run] omit = none +[flake8] +exclude = .ipynb_checkpoints +ignore = E501,W504,W503 + [testenv] # install pytest in the virtualenv where commands will be executed deps = pytest flake8 - coverage commands = # NOTE: you can run any command line tool here - not just tests - flake8 --ignore=W503,W504 src/PseudoNetCDF - coverage run -m pytest - coverage report -i + flake8 src/PseudoNetCDF + python -m pytest -v src/PseudoNetCDF [gh-actions] python = From 26803af16a9d52db1f76910cf4c07558e9496643 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:50:41 -0500 Subject: [PATCH 4/8] flake8 errors flake8 does not like explicit global statements if the variable is never defined within a function. --- src/PseudoNetCDF/_getreader.py | 2 -- src/PseudoNetCDF/_getwriter.py | 1 - src/PseudoNetCDF/plotutil/colors.py | 2 -- src/PseudoNetCDF/pncwarn.py | 1 - 4 files changed, 6 deletions(-) diff --git a/src/PseudoNetCDF/_getreader.py b/src/PseudoNetCDF/_getreader.py index cf2af7a..0061709 100644 --- a/src/PseudoNetCDF/_getreader.py +++ b/src/PseudoNetCDF/_getreader.py @@ -31,7 +31,6 @@ def getreader(*args, **kwds): ------- reader : class """ - global _readers format = kwds.pop('format', None) if not os.path.isfile(args[0]): warn(('The first argument (%s) does not exist as a file. ' + @@ -71,7 +70,6 @@ def checker(*args, **kwds): def registerreader(name, reader): - global _readers, pncopen if name not in [k for k, v in _readers]: _readers.insert(0, (name, reader)) if pncopen.__doc__ is not None: diff --git a/src/PseudoNetCDF/_getwriter.py b/src/PseudoNetCDF/_getwriter.py index 88f3888..a85170e 100644 --- a/src/PseudoNetCDF/_getwriter.py +++ b/src/PseudoNetCDF/_getwriter.py @@ -12,7 +12,6 @@ def testwriter(writer, *args, **kwds): def registerwriter(name, writer): - global _writers _writers.insert(0, (name, writer)) diff --git a/src/PseudoNetCDF/plotutil/colors.py b/src/PseudoNetCDF/plotutil/colors.py index ee3453c..6e86f79 100644 --- a/src/PseudoNetCDF/plotutil/colors.py +++ b/src/PseudoNetCDF/plotutil/colors.py @@ -14,12 +14,10 @@ def get_norm(name): - global _registered_norms return _registered_norms[name] def register_norm(name, norm): - global _registered_norms if isinstance(norm, Normalize): _registered_norms[name] = norm else: diff --git a/src/PseudoNetCDF/pncwarn.py b/src/PseudoNetCDF/pncwarn.py index cfbb8bb..37321af 100644 --- a/src/PseudoNetCDF/pncwarn.py +++ b/src/PseudoNetCDF/pncwarn.py @@ -28,7 +28,6 @@ def warn(*args, **kwds): def clean_showwarning(message, category, filename, lineno, file=None, line=None): - global _first_read_only if file is None: file = sys.stderr if file is None: From 923f65af91e6a3b735d0c7fa12e03667f8630289 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:51:05 -0500 Subject: [PATCH 5/8] adding more descriptive error around packaging --- src/PseudoNetCDF/test/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/PseudoNetCDF/test/__init__.py b/src/PseudoNetCDF/test/__init__.py index 8592ca0..404ca73 100644 --- a/src/PseudoNetCDF/test/__init__.py +++ b/src/PseudoNetCDF/test/__init__.py @@ -4,7 +4,12 @@ def _importorskip(modname, minversion=None): try: from packaging.version import Version except Exception: - from distutils.version import LooseVersion as Version + msg = 'packaging (python>=3.8) was not available.' + try: + from distutils.version import LooseVersion as Version + except Exception: + msg += ' distutils (python<3.12) was not available' + raise ImportError(msg) try: mod = importlib.import_module(modname) has = True From 3a979f82ee4a055095f186ae0bb472c241ed638b Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:51:21 -0500 Subject: [PATCH 6/8] using context for tempdirectory --- src/PseudoNetCDF/test/test_core.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/PseudoNetCDF/test/test_core.py b/src/PseudoNetCDF/test/test_core.py index 0b2301f..e947bf4 100644 --- a/src/PseudoNetCDF/test/test_core.py +++ b/src/PseudoNetCDF/test/test_core.py @@ -936,16 +936,17 @@ def testOpenMFDataset(self): tncf = self.testncf.copy() to3 = tncf.variables['O3'][:] - tmpdirname = tempfile.gettempdir() - tmppath1 = os.path.join(tmpdirname, 'test1.nc') - tmppath2 = os.path.join(tmpdirname, 'test2.nc') - tncf.save(tmppath1).close() - tncf.save(tmppath2).close() - nt = len(tncf.dimensions['TSTEP']) - cncf = netcdf.open_mfdataset(tmppath1, tmppath2, stackdim='TSTEP') - np_all_close(cncf.variables['O3'][:nt], to3) - np_all_close(cncf.variables['O3'][nt:], to3) - cncf.close() + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = '.' + tmppath1 = os.path.join(tmpdirname, 'test3.nc') + tmppath2 = os.path.join(tmpdirname, 'test4.nc') + tncf.save(tmppath1).close() + tncf.save(tmppath2).close() + nt = len(tncf.dimensions['TSTEP']) + cncf = netcdf.open_mfdataset(tmppath1, tmppath2, stackdim='TSTEP') + np_all_close(cncf.variables['O3'][:nt], to3) + np_all_close(cncf.variables['O3'][nt:], to3) + cncf.close() # to ensure compatibility with windows, removing any references to # underlying files so they can be deleted. del cncf From fcea51ab041bc762e03b1d635f0ec80b350489d8 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 11:56:20 -0500 Subject: [PATCH 7/8] add explicit v3 versiosn --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 6962f26..febc6b4 100644 --- a/setup.py +++ b/setup.py @@ -109,6 +109,10 @@ def find_data(): url='http://github.com/barronh/pseudonetcdf/', classifiers=[ 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Operating System :: MacOS', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', From 171d350eda3c2daa808d9dca592c5f89e57f27c9 Mon Sep 17 00:00:00 2001 From: barronh Date: Mon, 23 Feb 2026 12:40:56 -0500 Subject: [PATCH 8/8] testOpenMFDataset is flaky with specific netcdf versions --- src/PseudoNetCDF/test/test_core.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/PseudoNetCDF/test/test_core.py b/src/PseudoNetCDF/test/test_core.py index e947bf4..a3a541b 100644 --- a/src/PseudoNetCDF/test/test_core.py +++ b/src/PseudoNetCDF/test/test_core.py @@ -927,6 +927,7 @@ def testNetcdf(self): del cncf os.remove(tmppath) + @unittest.skip('Flaky with parallel netCDF4>=1.6') def testOpenMFDataset(self): import tempfile import os @@ -937,26 +938,27 @@ def testOpenMFDataset(self): tncf = self.testncf.copy() to3 = tncf.variables['O3'][:] with tempfile.TemporaryDirectory() as tmpdirname: - tmpdirname = '.' - tmppath1 = os.path.join(tmpdirname, 'test3.nc') - tmppath2 = os.path.join(tmpdirname, 'test4.nc') + tmppath1 = os.path.join(tmpdirname, 'test1.nc') + tmppath2 = os.path.join(tmpdirname, 'test2.nc') tncf.save(tmppath1).close() tncf.save(tmppath2).close() nt = len(tncf.dimensions['TSTEP']) + cncf = netcdf.open_mfdataset(tmppath1, tmppath2, stackdim='TSTEP') np_all_close(cncf.variables['O3'][:nt], to3) np_all_close(cncf.variables['O3'][nt:], to3) cncf.close() - # to ensure compatibility with windows, removing any references to - # underlying files so they can be deleted. - del cncf - gc.collect() - try: - os.remove(tmppath1) - os.remove(tmppath2) - except Exception: - warnings.warn(f'Could not delete {tmppath1} and {tmppath2}') - pass + + # to ensure compatibility with windows, removing any references to + # underlying files so they can be deleted. + del cncf + gc.collect() + try: + os.remove(tmppath1) + os.remove(tmppath2) + except Exception: + warnings.warn(f'Could not delete {tmppath1} and {tmppath2}') + pass def runTest(self): pass