From cdc25c6d9acf065733aba96305166d9ee6f4f5f3 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 26 Aug 2026 14:06:08 -0500 Subject: [PATCH 1/4] Add coll access when putting vars using nc4 Adding collective access when writing/putting vars using NetCDF4 parallel I/O. The collective access is already set when defining these variables. However it was not set when a file is re-opened and the variable was appended (variable id is queried here) Also adding a util function to check if a variable has unlimited dimension. This function is currently not being used but is useful for debugging (and for future code development) --- src/clib/core/pio_getput_int.cpp | 46 ++++++++++++++++++++++++++++++++ src/clib/core/pio_nc.cpp | 5 ++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/clib/core/pio_getput_int.cpp b/src/clib/core/pio_getput_int.cpp index c763db1b8c..6caece2915 100644 --- a/src/clib/core/pio_getput_int.cpp +++ b/src/clib/core/pio_getput_int.cpp @@ -12,6 +12,8 @@ #include "spio_io_summary.h" #include "spio_hash.h" #include "spio_hdf5_utils.hpp" +#include +#include #if PIO_USE_ASYNC_WR_THREAD #include "spio_async_hdf5_utils.hpp" #endif @@ -2331,6 +2333,39 @@ int spio_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) return spio_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf); } +/* Returns true if a variable has unlimited dimensions, false otherwise (and on error) */ +static bool spio_var_has_unlimited_dims(file_desc_t *file, int varid, int &ret) +{ + ret = PIO_NOERR; + assert(file && (varid >= 0)); + + /* Get the unlimited dimension in the file + * Note: We assume that we have just one unlimited dimension (no longer valid for + * NetCDF4 files) in a file + */ + int unlim_dimid = -1; + ret = PIOc_inq_unlimdim_impl(file->pio_ncid, &unlim_dimid); + if((ret != PIO_NOERR) || (unlim_dimid == -1)) { return false; } + + /* Get the number of dimensions in the variable */ + int ndims = -1; + ret = PIOc_inq_varndims_impl(file->pio_ncid, varid, &ndims); + if((ret != PIO_NOERR) || (ndims <= 0)) { return false; } + + /* Get the variable dimensions */ + std::vector vdims(ndims); + ret = PIOc_inq_vardimid_impl(file->pio_ncid, varid, vdims.data()); + if(ret != PIO_NOERR) { return false; } + + /* Check if any variable dimension is an unlimited dimension */ + std::vector::const_iterator iter = + std::find_if(vdims.cbegin(), vdims.cend(), + [unlim_dimid](int vdim){ return vdim == unlim_dimid; }); + if(iter != vdims.cend()) { return true; } + + return false; +} + /** * Internal PIO function which provides a type-neutral interface to * nc_put_vars. @@ -3309,6 +3344,17 @@ int spio_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC && file->iotype != PIO_IOTYPE_HDF5 && file->iotype != PIO_IOTYPE_HDF5C && file->do_io) { +#ifdef _NETCDF4 + /* If using NetCDF4 parallel I/O set the access mode to COLLECTIVE */ + if((file->iotype == PIO_IOTYPE_NETCDF4P) || (file->iotype == PIO_IOTYPE_NETCDF4P_NCZARR)){ + /* FIXME: We might just need this for vars with unlimited dims + * i.e., spio_var_has_unlimited_dims(file, varid, ierr) == true + * However to be consistent with PIOc_def_var() setting it for all vars + */ + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + } +#endif /* _NETCDF4 */ + LOG((2, "spio_put_vars_tc calling netcdf function file->iotype = %d", file->iotype)); ios->io_fstats->wb += num_elem * typelen; diff --git a/src/clib/core/pio_nc.cpp b/src/clib/core/pio_nc.cpp index b0c700594e..d32a183208 100644 --- a/src/clib/core/pio_nc.cpp +++ b/src/clib/core/pio_nc.cpp @@ -1777,8 +1777,9 @@ int PIOc_inq_varid_impl(int ncid, const char *name, int *varidp) #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC && file->do_io) - ierr = nc_inq_varid(file->fh, name, varidp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC && file->do_io){ + ierr = nc_inq_varid(file->fh, name, varidp); + } #endif /* _NETCDF */ } From 41f60af0f1b49f4218572423ab216391a86047d3 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 26 Aug 2026 23:21:52 -0500 Subject: [PATCH 2/4] Cleanup after using HDF5 and NetCDF libs Adding functions to cleanup internal memory/objects in NetCDF and HDF5 libraries --- src/clib/core/pioc.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/clib/core/pioc.cpp b/src/clib/core/pioc.cpp index b3329c6460..fdd5f8fc45 100644 --- a/src/clib/core/pioc.cpp +++ b/src/clib/core/pioc.cpp @@ -1747,6 +1747,18 @@ int PIOc_finalize_impl(int iosysid) } } #endif +#ifdef _NETCDF + if(niosysid == 1){ + /* All data should already be flushed by now. Free any NetCDF internal structures/objects/memory */ + nc_finalize(); + } +#endif +#ifdef _HDF5 + if(niosysid == 1){ + /* All data should already be flushed by now. Free any HDF5 internal structures/objects/memory */ + H5close(); + } +#endif LOG((2, "%d iosystems are still open.", niosysid)); free(ios->io_fstats); From efc3f0688643fa9fee9ba8af0e90faf958472e74 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 27 Aug 2026 10:07:55 -0500 Subject: [PATCH 3/4] Close unclosed files on finalize Make sure that unclosed files are closed on finalize. Moving the function out of async block so that it runs for non-async cases as well. Also update the function to handle async and non-async cases appropriately --- src/clib/core/pioc.cpp | 8 ++++++-- src/clib/core/util/pio_lists.cpp | 14 ++++++++------ src/clib/pio_internal.h | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/clib/core/pioc.cpp b/src/clib/core/pioc.cpp index fdd5f8fc45..45021b3327 100644 --- a/src/clib/core/pioc.cpp +++ b/src/clib/core/pioc.cpp @@ -1701,13 +1701,17 @@ int PIOc_finalize_impl(int iosysid) } } + /* Close files that were not explicitly closed by the user */ + bool wait_on_hard_close = false; #if PIO_USE_ASYNC_WR_THREAD - ierr = spio_close_all_files_and_delete_from_list(iosysid); + /* Wait on pending async ops on file before closing it */ + wait_on_hard_close = true; +#endif + ierr = spio_close_all_files_and_delete_from_list(iosysid, wait_on_hard_close); if(ierr != PIO_NOERR){ return pio_err(ios, NULL, ierr, __FILE__, __LINE__, "PIO Finalize failed on iosytem (%d). Error closing files on this I/O system", iosysid); } -#endif #if PIO_SAVE_DECOMPS SPIO_Util::Decomp_Util::serialize_decomp_map_info_pool(ios); diff --git a/src/clib/core/util/pio_lists.cpp b/src/clib/core/util/pio_lists.cpp index c85ed212da..89488cd4c5 100644 --- a/src/clib/core/util/pio_lists.cpp +++ b/src/clib/core/util/pio_lists.cpp @@ -213,7 +213,7 @@ int pio_delete_file_from_list(int ncid) return pio_free_file(file); } -int spio_close_all_files_and_delete_from_list(int iosysid) +int spio_close_all_files_and_delete_from_list(int iosysid, bool wait_on_hard_close) { int ret = PIO_NOERR; std::vector ncids_to_del_from_list; @@ -222,11 +222,13 @@ int spio_close_all_files_and_delete_from_list(int iosysid) file_desc_t *file = iter->second; assert(file); if(file->iosystem->iosysid == iosysid){ - //ret = spio_hard_closefile(file->iosystem, file, true); - ret = spio_wait_on_hard_close(file->iosystem, file); - if(ret != PIO_NOERR){ - return pio_err(file->iosystem, file, PIO_EINTERNAL, __FILE__, __LINE__, - "Error closing file (hard close failed)"); + if(wait_on_hard_close){ + //ret = spio_hard_closefile(file->iosystem, file, true); + ret = spio_wait_on_hard_close(file->iosystem, file); + if(ret != PIO_NOERR){ + return pio_err(file->iosystem, file, PIO_EINTERNAL, __FILE__, __LINE__, + "Error closing file (hard close failed)"); + } } ncids_to_del_from_list.push_back(file->pio_ncid); } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 57dd77006c..53b279e4ba 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -123,7 +123,7 @@ extern "C" { int pio_get_file(int ncid, file_desc_t **filep); int pio_delete_file_from_list(int ncid); - int spio_close_all_files_and_delete_from_list(int iosysid); + int spio_close_all_files_and_delete_from_list(int iosysid, bool wait_on_hard_close); int spio_close_soft_closed_file(const char *filename); int pio_add_to_file_list(file_desc_t *file, MPI_Comm comm); From d0ecccef680f2f75c47ca2baa924a2b2a845f649 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 27 Aug 2026 10:34:57 -0500 Subject: [PATCH 4/4] Close/Delete decomps in finalize Make sure that all I/O decomps are finalized/deleted in finalize. Moving the call to delete I/O decomps outside "async block" of code so that it also runs for non-async cases. I/O decomps contain MPI types and required MPI calls for cleanup, so destructors cannot automatically clean it up (since the destructors will end up being called after MPI finalize). --- src/clib/core/pioc.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/clib/core/pioc.cpp b/src/clib/core/pioc.cpp index 45021b3327..c4cadb9bb9 100644 --- a/src/clib/core/pioc.cpp +++ b/src/clib/core/pioc.cpp @@ -1742,15 +1742,17 @@ int PIOc_finalize_impl(int iosysid) "PIO Finalize failed on iosystem (%d). Unable to get the number of open I/O systems", iosysid); } -#if PIO_USE_ASYNC_WR_THREAD if(niosysid == 1){ + /* All data should be flushed by now, so I/O decomps are no longer needed. Delete all I/O decomps. + * Note: Some I/O decomps may not be freed/deleted on PIOc_freedecomp() due to pending rearrangement/writes + */ ierr = pio_delete_all_iodescs(iosysid); if(ierr != PIO_NOERR){ return pio_err(ios, NULL, ierr, __FILE__, __LINE__, "PIO Finalize failed on iosytem (%d). Error deleting I/O decomps on this I/O system", iosysid); } } -#endif + #ifdef _NETCDF if(niosysid == 1){ /* All data should already be flushed by now. Free any NetCDF internal structures/objects/memory */