Dealing with #1004, #1005, #1006

hera/datalayer/datahandler.py

@@ -1,9 +1,9 @@
+import importlib
 import json
 import pickle
-import importlib
-import os
-from hera.utils.lazy import _LazyModule
 
+from hera.utils.jsonutils import ConfigurationToJSON, JSONToConfiguration
+from hera.utils.lazy import _LazyModule
 
 numpy  = _LazyModule("numpy")
 pandas = _LazyModule("pandas")
@@ -383,10 +383,13 @@ def getData(resource, usePandas=False, desc=None):
 
 class DataHandler_netcdf_xarray(object):
     """Handler for xarray datasets stored as NetCDF files."""
+    SERIALIZED_ATTRS_KEY = "_serialized_attrs"
 
     @staticmethod
     def saveData(resource, fileName,**kwargs):
         """Save an xarray dataset to NetCDF format."""
+        flattened_attrs = ConfigurationToJSON(resource.attrs)
+        resource.attrs = {DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY:json.dumps(flattened_attrs)}
         resource.to_netcdf(fileName,**kwargs)
         return dict()
 
@@ -408,9 +411,12 @@ def getData(resource, desc=None, **kwargs):
         xarray
         """
         import xarray
-        df = xarray.open_mfdataset(resource, combine='by_coords', **kwargs)
+        ds = xarray.open_mfdataset(resource, combine='by_coords', **kwargs)
+        if DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY in ds.attrs:
+            ds.attrs = json.loads(ds.attrs[DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY])
+        ds.attrs = JSONToConfiguration(ds.attrs)
 
-        return df
+        return ds
 
 
 class DataHandler_zarr_xarray(object):
@@ -430,6 +436,7 @@ def saveData(resource, fileName,**kwargs):
         -------
 
         """
+        resource.attrs = ConfigurationToJSON(resource.attrs)
         resource.to_zarr(fileName, mode="w",**kwargs)
         return dict()
 
@@ -452,6 +459,7 @@ def getData(resource, desc=None, **kwargs):
         """
         import xarray
         df = xarray.open_zarr(resource, **kwargs)
+        resource.attrs = JSONToConfiguration(resource.attrs)
         return df
 
 
@@ -545,6 +553,7 @@ def saveData(resource, fileName,**kwargs):
     def getData(resource, desc=None, **kwargs):
         """Load a GeoDataFrame from a GeoJSON file."""
         import geopandas
+
         from hera.utils.jsonutils import loadJSON
         desc = desc or {}
         df = geopandas.GeoDataFrame.from_features(loadJSON(resource)["features"])
@@ -842,9 +851,9 @@ def getData(resource, desc=None, **kwargs):
         Uses importlib.util.spec_from_file_location for resource-based loading
         so that sys.path is never mutated by DB-supplied paths [1.6, 3.3].
         """
-        import os
         import importlib
         import importlib.util
+        import os
 
         # 1) Resolve metadata
         desc = desc or {}

hera/simulations/LSM/singleSimulation.py

@@ -1,10 +1,9 @@
 import os
-import xarray
+
 import numpy
-import os
-from hera.utils.unitHandler import ureg, unumToPint
+import xarray
 
-from ...utils import tounit,tonumber
+from hera.utils.unitHandler import unumToPint, ureg
 
 
 class SingleSimulation(object):
@@ -19,18 +18,19 @@ def params(self):
     def version(self):
         return self._document['desc']['version']
 
-    def __init__(self, resource):
-
+    def __init__(self, resource, chunks={'datetime':-1, 'x':'auto', 'y':'auto', 'z':'auto'}):
         if isinstance(resource,str):
             try:
-                self._finalxarray = xarray.open_mfdataset(os.path.join(resource, '*.nc'), combine='by_coords')
+                self._finalxarray = xarray.open_mfdataset(os.path.join(resource, '*.nc'), combine='by_coords', chunks=chunks)
             except OSError:
-                self._finalxarray = xarray.open_mfdataset(os.path.join(resource,"netcdf", '*.nc'), combine='by_coords')
+                self._finalxarray = xarray.open_mfdataset(os.path.join(resource,"netcdf", '*.nc'), combine='by_coords', chunks=chunks)
+        elif isinstance(resource, (xarray.DataArray, xarray.Dataset)):
+            self._finalxarray=resource
         else:
             self._document = resource
             self._finalxarray = resource.getData()
             if type(self._finalxarray) is str:
-                self._finalxarray = xarray.open_mfdataset(self._finalxarray, combine='by_coords')
+                self._finalxarray = xarray.open_mfdataset(self._finalxarray, combine='by_coords', chunks=chunks)
 
     def getDosage(self, Q=1 * ureg.kg, time_units=ureg.min, q_units=ureg.mg):
         """
@@ -58,12 +58,16 @@ def getDosage(self, Q=1 * ureg.kg, time_units=ureg.min, q_units=ureg.mg):
         Q = unumToPint(Q)
         time_units = unumToPint(time_units)
         q_units = unumToPint(q_units)
+        from pandas.api.types import is_numeric_dtype
 
         final_xarray = self._finalxarray.copy()
-        if type(final_xarray.datetime.diff('datetime')[0].values.item())==float:
-            dt_minutes = final_xarray.datetime.diff('datetime')[0].values.item()*ureg.sec #temporary solution!!!!!
+        if is_numeric_dtype(final_xarray.datetime.dtype):
+            dt_minutes = final_xarray.datetime.isel(datetime=[0,1]).diff('datetime')[0].values.item()*ureg.sec
+            # old solution:
+            #   type(final_xarray.datetime.diff('datetime')[0].values.item()) is float:
+            #   .diff('datetime')[0].values.item()*ureg.sec #temporary solution!!!!!
         else:
-            dt_minutes = (final_xarray.datetime.diff('datetime')[0].values / numpy.timedelta64(1, 'm')) * ureg.min
+            dt_minutes = (final_xarray.datetime.isel(datetime=[0,1]).diff('datetime')[0].values / numpy.timedelta64(1, 'm')) * ureg.min
         final_xarray.attrs['dt'] = dt_minutes.to(time_units)
         final_xarray.attrs['Q']  = Q.to(q_units)
         final_xarray.attrs['C']  = q_units/ ureg.m ** 3

hera/simulations/hermesWorkflowToolkit.py

@@ -809,7 +809,7 @@ def executeWorkflowFromDB(self, nameOrWorkflowFileOrJSONOrResource,
             # hermes.build() traverses the workflow node tree, wraps each node in a
             # Luigi task, and returns the Python source code for the task module.
             logger.info(f"Building and executing the workflow {workflowName}")
-            build = hermesWF.build(buildername=workflow.BUILDER_LUIGI, dispatch_id=dispatch_id)
+            build = hermesWF.build(buildername=workflow.BUILDER_LUIGI)
 
             # Step 3: Write the workflow JSON and generated Python module to disk.
             # The JSON is written to the resource path; the Python module contains