Fix issues with writing vars with unlimited dimensions using NetCDF/HDF5

src/clib/core/pio_getput_int.cpp

@@ -12,6 +12,8 @@
 #include "spio_io_summary.h"
 #include "spio_hash.h"
 #include "spio_hdf5_utils.hpp"
+#include <vector>
+#include <algorithm>
 #if PIO_USE_ASYNC_WR_THREAD
 #include "spio_async_hdf5_utils.hpp"
 #endif
@@ -2331,6 +2333,39 @@ int spio_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
     return spio_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf);
 }
 
+/* Returns true if a variable has unlimited dimensions, false otherwise (and on error) */
+static bool spio_var_has_unlimited_dims(file_desc_t *file, int varid, int &ret)
+{
+  ret = PIO_NOERR;
+  assert(file && (varid >= 0));
+
+  /* Get the unlimited dimension in the file
+   * Note: We assume that we have just one unlimited dimension (no longer valid for
+   * NetCDF4 files) in a file
+   */
+  int unlim_dimid = -1;
+  ret = PIOc_inq_unlimdim_impl(file->pio_ncid, &unlim_dimid);
+  if((ret != PIO_NOERR) || (unlim_dimid == -1)) { return false; }
+
+  /* Get the number of dimensions in the variable */
+  int ndims = -1;
+  ret = PIOc_inq_varndims_impl(file->pio_ncid, varid, &ndims);
+  if((ret != PIO_NOERR) || (ndims <= 0)) { return false; }
+
+  /* Get the variable dimensions */
+  std::vector<int> vdims(ndims);
+  ret = PIOc_inq_vardimid_impl(file->pio_ncid, varid, vdims.data());
+  if(ret != PIO_NOERR) { return false; }
+
+  /* Check if any variable dimension is an unlimited dimension */
+  std::vector<int>::const_iterator iter =
+    std::find_if(vdims.cbegin(), vdims.cend(),
+      [unlim_dimid](int vdim){ return vdim == unlim_dimid; });
+  if(iter != vdims.cend()) { return true; }
+
+  return false;
+}
+
 /**
  * Internal PIO function which provides a type-neutral interface to
  * nc_put_vars.
@@ -3309,6 +3344,17 @@ int spio_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
             file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC &&
             file->iotype != PIO_IOTYPE_HDF5 && file->iotype != PIO_IOTYPE_HDF5C && file->do_io)
         {
+#ifdef _NETCDF4
+          /* If using NetCDF4 parallel I/O set the access mode to COLLECTIVE */
+          if((file->iotype == PIO_IOTYPE_NETCDF4P) || (file->iotype == PIO_IOTYPE_NETCDF4P_NCZARR)){
+            /* FIXME: We might just need this for vars with unlimited dims
+             * i.e., spio_var_has_unlimited_dims(file, varid, ierr) == true
+             * However to be consistent with PIOc_def_var() setting it for all vars
+             */
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+          }
+#endif /* _NETCDF4 */
+
             LOG((2, "spio_put_vars_tc calling netcdf function file->iotype = %d",
                  file->iotype));
             ios->io_fstats->wb += num_elem * typelen;

src/clib/core/pio_nc.cpp

@@ -1777,8 +1777,9 @@ int PIOc_inq_varid_impl(int ncid, const char *name, int *varidp)
 #endif /* _PNETCDF */
 
 #ifdef _NETCDF
-        if (file->iotype != PIO_IOTYPE_PNETCDF && file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC && file->do_io)
-            ierr = nc_inq_varid(file->fh, name, varidp);
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->iotype != PIO_IOTYPE_ADIOS && file->iotype != PIO_IOTYPE_ADIOSC && file->do_io){
+          ierr = nc_inq_varid(file->fh, name, varidp);
+        }
 #endif /* _NETCDF */
     }
 

src/clib/core/pioc.cpp

@@ -1701,13 +1701,17 @@ int PIOc_finalize_impl(int iosysid)
         }
     }
 
+    /* Close files that were not explicitly closed by the user */
+    bool wait_on_hard_close = false;
 #if PIO_USE_ASYNC_WR_THREAD
-    ierr = spio_close_all_files_and_delete_from_list(iosysid);
+    /* Wait on pending async ops on file before closing it */
+    wait_on_hard_close = true;
+#endif
+    ierr = spio_close_all_files_and_delete_from_list(iosysid, wait_on_hard_close);
     if(ierr != PIO_NOERR){
       return pio_err(ios, NULL, ierr, __FILE__, __LINE__,
                       "PIO Finalize failed on iosytem (%d). Error closing files on this I/O system", iosysid);
     }
-#endif
 
 #if PIO_SAVE_DECOMPS
     SPIO_Util::Decomp_Util::serialize_decomp_map_info_pool(ios);
@@ -1738,14 +1742,28 @@ int PIOc_finalize_impl(int iosysid)
                         "PIO Finalize failed on iosystem (%d). Unable to get the number of open I/O systems", iosysid);
     }
 
-#if PIO_USE_ASYNC_WR_THREAD
     if(niosysid == 1){
+      /* All data should be flushed by now, so I/O decomps are no longer needed. Delete all I/O decomps.
+       * Note: Some I/O decomps may not be freed/deleted on PIOc_freedecomp() due to pending rearrangement/writes
+       */
       ierr = pio_delete_all_iodescs(iosysid);
       if(ierr != PIO_NOERR){
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__,
                         "PIO Finalize failed on iosytem (%d). Error deleting I/O decomps on this I/O system", iosysid);
       }
     }
+
+#ifdef _NETCDF
+    if(niosysid == 1){
+      /* All data should already be flushed by now. Free any NetCDF internal structures/objects/memory */
+      nc_finalize();
+    }
+#endif
+#ifdef _HDF5
+    if(niosysid == 1){
+      /* All data should already be flushed by now. Free any HDF5 internal structures/objects/memory */
+      H5close();
+    }
 #endif
 
     LOG((2, "%d iosystems are still open.", niosysid));

src/clib/core/util/pio_lists.cpp

@@ -213,7 +213,7 @@ int pio_delete_file_from_list(int ncid)
   return pio_free_file(file);
 }
 
-int spio_close_all_files_and_delete_from_list(int iosysid)
+int spio_close_all_files_and_delete_from_list(int iosysid, bool wait_on_hard_close)
 {
   int ret = PIO_NOERR;
   std::vector<int> ncids_to_del_from_list;
@@ -222,11 +222,13 @@ int spio_close_all_files_and_delete_from_list(int iosysid)
     file_desc_t *file = iter->second;
     assert(file);
     if(file->iosystem->iosysid == iosysid){
-      //ret = spio_hard_closefile(file->iosystem, file, true);
-      ret = spio_wait_on_hard_close(file->iosystem, file);
-      if(ret != PIO_NOERR){
-        return pio_err(file->iosystem, file, PIO_EINTERNAL, __FILE__, __LINE__,
-                        "Error closing file (hard close failed)");
+      if(wait_on_hard_close){
+        //ret = spio_hard_closefile(file->iosystem, file, true);
+        ret = spio_wait_on_hard_close(file->iosystem, file);
+        if(ret != PIO_NOERR){
+          return pio_err(file->iosystem, file, PIO_EINTERNAL, __FILE__, __LINE__,
+                          "Error closing file (hard close failed)");
+        }
       }
       ncids_to_del_from_list.push_back(file->pio_ncid);
     }

src/clib/pio_internal.h

@@ -123,7 +123,7 @@ extern "C" {
 
     int pio_get_file(int ncid, file_desc_t **filep);
     int pio_delete_file_from_list(int ncid);
-    int spio_close_all_files_and_delete_from_list(int iosysid);
+    int spio_close_all_files_and_delete_from_list(int iosysid, bool wait_on_hard_close);
     int spio_close_soft_closed_file(const char *filename);
     int pio_add_to_file_list(file_desc_t *file, MPI_Comm comm);