From 802fbb2aad4221d29fee32265699e54415e9e025 Mon Sep 17 00:00:00 2001 From: LiorAntonov Date: Wed, 22 Jul 2026 10:24:27 +0300 Subject: [PATCH] Dealing with #1004, #1005, #1006 Updating Hermes along the way --- Hermes | 2 +- hera/datalayer/datahandler.py | 21 ++++++++++++----- hera/simulations/LSM/singleSimulation.py | 28 +++++++++++++---------- hera/simulations/hermesWorkflowToolkit.py | 2 +- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Hermes b/Hermes index 8f066f7c4..f1f547117 160000 --- a/Hermes +++ b/Hermes @@ -1 +1 @@ -Subproject commit 8f066f7c40b36ca9c94074e805c605a1053bf503 +Subproject commit f1f547117b8c23eccf60d8c0e1bd1573803e858d diff --git a/hera/datalayer/datahandler.py b/hera/datalayer/datahandler.py index b4a04abd3..925189228 100644 --- a/hera/datalayer/datahandler.py +++ b/hera/datalayer/datahandler.py @@ -1,9 +1,9 @@ +import importlib import json import pickle -import importlib -import os -from hera.utils.lazy import _LazyModule +from hera.utils.jsonutils import ConfigurationToJSON, JSONToConfiguration +from hera.utils.lazy import _LazyModule numpy = _LazyModule("numpy") pandas = _LazyModule("pandas") @@ -383,10 +383,13 @@ def getData(resource, usePandas=False, desc=None): class DataHandler_netcdf_xarray(object): """Handler for xarray datasets stored as NetCDF files.""" + SERIALIZED_ATTRS_KEY = "_serialized_attrs" @staticmethod def saveData(resource, fileName,**kwargs): """Save an xarray dataset to NetCDF format.""" + flattened_attrs = ConfigurationToJSON(resource.attrs) + resource.attrs = {DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY:json.dumps(flattened_attrs)} resource.to_netcdf(fileName,**kwargs) return dict() @@ -408,9 +411,12 @@ def getData(resource, desc={}, **kwargs): xarray """ import xarray - df = xarray.open_mfdataset(resource, combine='by_coords', **kwargs) + ds = xarray.open_mfdataset(resource, combine='by_coords', **kwargs) + if DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY in ds.attrs: + ds.attrs = json.loads(ds.attrs[DataHandler_netcdf_xarray.SERIALIZED_ATTRS_KEY]) + ds.attrs = JSONToConfiguration(ds.attrs) - return df + return ds class DataHandler_zarr_xarray(object): @@ -430,6 +436,7 @@ def saveData(resource, fileName,**kwargs): ------- """ + resource.attrs = ConfigurationToJSON(resource.attrs) resource.to_zarr(fileName, mode="w",**kwargs) return dict() @@ -452,6 +459,7 @@ def getData(resource, desc={}, **kwargs): """ import xarray df = xarray.open_zarr(resource, **kwargs) + resource.attrs = JSONToConfiguration(resource.attrs) return df @@ -545,6 +553,7 @@ def saveData(resource, fileName,**kwargs): def getData(resource, desc={}, **kwargs): """Load a GeoDataFrame from a GeoJSON file.""" import geopandas + from hera.utils.jsonutils import loadJSON df = geopandas.GeoDataFrame.from_features(loadJSON(resource)["features"]) if "crs" in desc: @@ -836,9 +845,9 @@ def saveData(resource, fileName, **kwargs): @staticmethod def getData(resource, desc=None, **kwargs): """Import and optionally instantiate a class from ``desc['classpath']``.""" + import importlib import os import sys - import importlib # 1) Add search paths to sys.path: # - If resource points to the package directory itself (contains __init__.py), diff --git a/hera/simulations/LSM/singleSimulation.py b/hera/simulations/LSM/singleSimulation.py index 3288fdf53..d78f65519 100644 --- a/hera/simulations/LSM/singleSimulation.py +++ b/hera/simulations/LSM/singleSimulation.py @@ -1,10 +1,9 @@ import os -import xarray + import numpy -import os -from hera.utils.unitHandler import ureg, unumToPint +import xarray -from ...utils import tounit,tonumber +from hera.utils.unitHandler import unumToPint, ureg class SingleSimulation(object): @@ -19,18 +18,19 @@ def params(self): def version(self): return self._document['desc']['version'] - def __init__(self, resource): - + def __init__(self, resource, chunks={'datetime':-1, 'x':'auto', 'y':'auto', 'z':'auto'}): if isinstance(resource,str): try: - self._finalxarray = xarray.open_mfdataset(os.path.join(resource, '*.nc'), combine='by_coords') + self._finalxarray = xarray.open_mfdataset(os.path.join(resource, '*.nc'), combine='by_coords', chunks=chunks) except OSError: - self._finalxarray = xarray.open_mfdataset(os.path.join(resource,"netcdf", '*.nc'), combine='by_coords') + self._finalxarray = xarray.open_mfdataset(os.path.join(resource,"netcdf", '*.nc'), combine='by_coords', chunks=chunks) + elif isinstance(resource, (xarray.DataArray, xarray.Dataset)): + self._finalxarray=resource else: self._document = resource self._finalxarray = resource.getData() if type(self._finalxarray) is str: - self._finalxarray = xarray.open_mfdataset(self._finalxarray, combine='by_coords') + self._finalxarray = xarray.open_mfdataset(self._finalxarray, combine='by_coords', chunks=chunks) def getDosage(self, Q=1 * ureg.kg, time_units=ureg.min, q_units=ureg.mg): """ @@ -58,12 +58,16 @@ def getDosage(self, Q=1 * ureg.kg, time_units=ureg.min, q_units=ureg.mg): Q = unumToPint(Q) time_units = unumToPint(time_units) q_units = unumToPint(q_units) + from pandas.api.types import is_numeric_dtype final_xarray = self._finalxarray.copy() - if type(final_xarray.datetime.diff('datetime')[0].values.item())==float: - dt_minutes = final_xarray.datetime.diff('datetime')[0].values.item()*ureg.sec #temporary solution!!!!! + if is_numeric_dtype(final_xarray.datetime.dtype): + dt_minutes = final_xarray.datetime.isel(datetime=[0,1]).diff('datetime')[0].values.item()*ureg.sec + # old solution: + # type(final_xarray.datetime.diff('datetime')[0].values.item()) is float: + # .diff('datetime')[0].values.item()*ureg.sec #temporary solution!!!!! else: - dt_minutes = (final_xarray.datetime.diff('datetime')[0].values / numpy.timedelta64(1, 'm')) * ureg.min + dt_minutes = (final_xarray.datetime.isel(datetime=[0,1]).diff('datetime')[0].values / numpy.timedelta64(1, 'm')) * ureg.min final_xarray.attrs['dt'] = dt_minutes.to(time_units) final_xarray.attrs['Q'] = Q.to(q_units) final_xarray.attrs['C'] = q_units/ ureg.m ** 3 diff --git a/hera/simulations/hermesWorkflowToolkit.py b/hera/simulations/hermesWorkflowToolkit.py index 0cf1d2828..3b4b9f468 100644 --- a/hera/simulations/hermesWorkflowToolkit.py +++ b/hera/simulations/hermesWorkflowToolkit.py @@ -804,7 +804,7 @@ def executeWorkflowFromDB(self, nameOrWorkflowFileOrJSONOrResource, # hermes.build() traverses the workflow node tree, wraps each node in a # Luigi task, and returns the Python source code for the task module. logger.info(f"Building and executing the workflow {workflowName}") - build = hermesWF.build(buildername=workflow.BUILDER_LUIGI, dispatch_id=dispatch_id) + build = hermesWF.build(buildername=workflow.BUILDER_LUIGI) # Step 3: Write the workflow JSON and generated Python module to disk. # The JSON is written to the resource path; the Python module contains