From aad0104c550ab6b0ffcee47ca2020e095e083bf3 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Mon, 18 May 2026 00:38:09 -0500
Subject: [PATCH 01/16] Get num threads from pool manager

Add interface to the thread pool manager to get the number of
threads (configured for the thread pool)
---
 src/clib/core/progress_engine/spio_async_tpool.cpp | 4 +---
 src/clib/core/progress_engine/spio_async_tpool.hpp | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/clib/core/progress_engine/spio_async_tpool.cpp b/src/clib/core/progress_engine/spio_async_tpool.cpp
index f96d271717..d5bd254637 100644
--- a/src/clib/core/progress_engine/spio_async_tpool.cpp
+++ b/src/clib/core/progress_engine/spio_async_tpool.cpp
@@ -89,11 +89,9 @@ int PIO_Util::PIO_async_tpool::dequeue_and_process(
 PIO_Util::PIO_async_tpool *
           PIO_Util::PIO_async_tpool_manager::get_tpool_instance(void )
 {
-  /* We need to make NUM_THREADS configurable by the user (compile-time) */
-  const int NUM_THREADS = SPIO_ASYNC_NTHREADS;
   if(tpool_ == NULL){
     LOG((2, "PIO_async_tpool_manager:get_tpool_instance: Creating new tpool instance"));
-    tpool_ = new PIO_Util::PIO_async_tpool(NUM_THREADS);
+    tpool_ = new PIO_Util::PIO_async_tpool(get_num_threads());
   }
   else{
     LOG((2, "PIO_async_tpool_manager:get_tpool_instance: Retrieving tpool instance"));
diff --git a/src/clib/core/progress_engine/spio_async_tpool.hpp b/src/clib/core/progress_engine/spio_async_tpool.hpp
index 8a6a9d0cf7..773a9c0e1e 100644
--- a/src/clib/core/progress_engine/spio_async_tpool.hpp
+++ b/src/clib/core/progress_engine/spio_async_tpool.hpp
@@ -27,6 +27,7 @@ class PIO_async_tpool{
 
 class PIO_async_tpool_manager{
   public:
+    static int get_num_threads(void) { return SPIO_ASYNC_NTHREADS; }
     static PIO_async_tpool *get_tpool_instance(void );
     ~PIO_async_tpool_manager();
   private:

From 06885927285644f290f9f6d8b079f97ef4c2b404 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Mon, 18 May 2026 00:39:39 -0500
Subject: [PATCH 02/16] Using tpool manager to get num threads

The compute processes do not require any threads. However these
processes need to know the number of threads (configured in
the thread pool - in the I/O procs)

On I/O processes use the thread pool manager to init the thread
pool and get the thread ids. On compute procs use the thread
pool manager to just get the number of threads.
---
 .../core/progress_engine/spio_async_tcomm.cpp  | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/clib/core/progress_engine/spio_async_tcomm.cpp b/src/clib/core/progress_engine/spio_async_tcomm.cpp
index 972f700257..b96e0bf5a4 100644
--- a/src/clib/core/progress_engine/spio_async_tcomm.cpp
+++ b/src/clib/core/progress_engine/spio_async_tcomm.cpp
@@ -17,8 +17,13 @@ bool SPIO_Util::TComm_info::is_thread_init_ = false;
 
 SPIO_Util::TComm_info::TComm_info(MPI_Comm union_comm, int union_comm_rank, int union_comm_io_root, int union_comm_comp_root, MPI_Comm io_comm, int io_comm_rank, bool is_io_master, MPI_Comm comp_comm, int comp_comm_rank, bool is_comp_master, MPI_Comm intercomm, MPI_Comm my_comm, MPI_Comm node_comm): union_comm_rank_(union_comm_rank), union_comm_io_root_(union_comm_io_root), union_comm_comp_root_(union_comm_comp_root), io_comm_rank_(io_comm_rank), is_io_master_(is_io_master), comp_comm_rank_(comp_comm_rank), is_comp_master_(is_comp_master)
 {
+  int nthreads = 1;
 #if PIO_USE_ASYNC_WR_THREAD
-  tids_ = PIO_Util::PIO_async_tpool_manager::get_tpool_instance()->get_thread_ids();  
+  nthreads += PIO_Util::PIO_async_tpool_manager::get_num_threads();
+  if(io_comm != MPI_COMM_NULL){
+    tids_ = PIO_Util::PIO_async_tpool_manager::get_tpool_instance()->get_thread_ids();
+    assert(static_cast<int>(tids_.size()) == nthreads - 1);
+  }
 #endif
 
   /* Total number of comms = Main/Default thread + Number of threads in thread pool
@@ -34,7 +39,7 @@ SPIO_Util::TComm_info::TComm_info(MPI_Comm union_comm, int union_comm_rank, int
   node_comms_.push_back(node_comm);
 
   int ret = MPI_SUCCESS;
-  for(std::size_t i = 0; i < tids_.size(); i++){
+  for(int i = 0; i < nthreads - 1; i++){
     MPI_Comm tmp_union_comm = MPI_COMM_NULL;
     if(union_comm != MPI_COMM_NULL){
       ret = MPI_Comm_dup(union_comm, &tmp_union_comm); assert(ret == MPI_SUCCESS);
@@ -135,10 +140,13 @@ SPIO_Util::TComm_info::~TComm_info()
    * Free other comms
    */
 
+  int nthreads = 1;
+#if PIO_USE_ASYNC_WR_THREAD
   /* Number of threads including the main thread */
-  std::size_t nthreads = tids_.size() + 1;
-  assert(union_comms_.size() == nthreads);
-  for(std::size_t tidx = 1; tidx < nthreads; tidx++){
+  nthreads += PIO_Util::PIO_async_tpool_manager::get_num_threads();
+#endif
+  assert(static_cast<int>(union_comms_.size()) == nthreads);
+  for(int tidx = 1; tidx < nthreads; tidx++){
     if(union_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&union_comms_[tidx]); }
     if(io_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&io_comms_[tidx]); }
     if(comp_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&comp_comms_[tidx]); }

From 6de9320364e96f75475221d825e30b3e1b6a20d3 Mon Sep 17 00:00:00 2001
From: Jayesh Krishna <jayesh@mcs.anl.gov>
Date: Thu, 21 May 2026 20:09:38 -0700
Subject: [PATCH 03/16] Disable waits for hdf5 ops on close

Disable waiting on pending HDF5 ops when closing non-HDF5 files
---
 src/clib/core/pio_file.cpp | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/clib/core/pio_file.cpp b/src/clib/core/pio_file.cpp
index 2d6739f062..a0f44564d9 100644
--- a/src/clib/core/pio_file.cpp
+++ b/src/clib/core/pio_file.cpp
@@ -1035,24 +1035,9 @@ int PIOc_closefile_impl(int ncid)
       if(ios->ioproc){
         soft_close = true;
       }
-      else{
-        /* Wait on all hdf5 async ops before a "hard close" */
-        /*
+      /* To wait on all hdf5 async ops before a "hard close" (for debugging)
         ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
-        if(ierr != PIO_NOERR){
-          return pio_err(ios, file, ierr, __FILE__, __LINE__,
-                          "Closing file (%s, ncid=%d) failed. Error sending async msg PIO_MSG_CLOSE_FILE", pio_get_fname_from_file(file), ncid);
-        }
-        */
-      }
-    }
-    else{
-      /* Wait on all hdf5 async ops before a "hard close" */
-      ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
-      if(ierr != PIO_NOERR){
-        return pio_err(ios, file, ierr, __FILE__, __LINE__,
-                        "Closing file (%s, ncid=%d) failed. Error sending async msg PIO_MSG_CLOSE_FILE", pio_get_fname_from_file(file), ncid);
-      }
+      */
     }
 #endif
 

From a4f7e9019db55688cc492b1889909901e4317f59 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Fri, 22 May 2026 16:26:08 -0500
Subject: [PATCH 04/16] Disable wait for pending ops on open

Disable waits for HDF5 ops when opening files
---
 src/clib/core/pioc_support.cpp | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/src/clib/core/pioc_support.cpp b/src/clib/core/pioc_support.cpp
index 655d341758..3d43f8865a 100644
--- a/src/clib/core/pioc_support.cpp
+++ b/src/clib/core/pioc_support.cpp
@@ -4583,14 +4583,9 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
                     "Creating file (%s) failed. Error closing previous soft closed file", filename);
   }
 
-  //if(((*iotype == PIO_IOTYPE_HDF5) || (*iotype == PIO_IOTYPE_HDF5C)) && !(mode & PIO_WRITE)){
-  if((*iotype == PIO_IOTYPE_HDF5) || (*iotype == PIO_IOTYPE_HDF5C)){
-    ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
-    if(ierr != PIO_NOERR){
-      return pio_err(ios, NULL, ierr, __FILE__, __LINE__,
-                      "Creating file (%s) failed. Error waiting on all pending asynchronous HDF5 ops", filename);
-    }
-  }
+  /* To wait on all hdf5 async ops (for debugging),
+   * ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
+   */
 #endif
 
   /* Allocate space for the file info. */

From 66bcfd92cf6f676f31d7f4a948ac4ebf9902607f Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Thu, 4 Jun 2026 15:58:54 -0500
Subject: [PATCH 05/16] Avoid duping unused comms for threads

Avoid duping comms that the I/O threads don't use (or should be using)
---
 .../core/progress_engine/spio_async_tcomm.cpp | 51 +++++++------------
 1 file changed, 17 insertions(+), 34 deletions(-)

diff --git a/src/clib/core/progress_engine/spio_async_tcomm.cpp b/src/clib/core/progress_engine/spio_async_tcomm.cpp
index b96e0bf5a4..5f16592a0c 100644
--- a/src/clib/core/progress_engine/spio_async_tcomm.cpp
+++ b/src/clib/core/progress_engine/spio_async_tcomm.cpp
@@ -40,11 +40,11 @@ SPIO_Util::TComm_info::TComm_info(MPI_Comm union_comm, int union_comm_rank, int
 
   int ret = MPI_SUCCESS;
   for(int i = 0; i < nthreads - 1; i++){
-    MPI_Comm tmp_union_comm = MPI_COMM_NULL;
-    if(union_comm != MPI_COMM_NULL){
-      ret = MPI_Comm_dup(union_comm, &tmp_union_comm); assert(ret == MPI_SUCCESS);
-    }
-    union_comms_.push_back(tmp_union_comm);
+    /* Since the I/O threads no longer have access (or should be using)
+     * to global communicators, except the I/O communicator (io_comm),
+     * all other communicators are assigned to COMM_NULL (not duped)
+     */
+    union_comms_.push_back(MPI_COMM_NULL);
 
     MPI_Comm tmp_io_comm = MPI_COMM_NULL;
     if(io_comm != MPI_COMM_NULL){
@@ -52,39 +52,22 @@ SPIO_Util::TComm_info::TComm_info(MPI_Comm union_comm, int union_comm_rank, int
     }
     io_comms_.push_back(tmp_io_comm);
 
-    MPI_Comm tmp_comp_comm = MPI_COMM_NULL;
-    if(comp_comm != MPI_COMM_NULL){
-      ret = MPI_Comm_dup(comp_comm, &tmp_comp_comm); assert(ret == MPI_SUCCESS);
-    }
-    comp_comms_.push_back(tmp_comp_comm);
-
-    MPI_Comm tmp_comm = MPI_COMM_NULL;
-    if(intercomm != MPI_COMM_NULL){
-      ret = MPI_Comm_dup(intercomm, &tmp_comm); assert(ret == MPI_SUCCESS);
-    }
-    intercomms_.push_back(tmp_comm);
-
+    comp_comms_.push_back(MPI_COMM_NULL);
+    intercomms_.push_back(MPI_COMM_NULL);
     /* my_comm, see iosystem_desc_t for details, is just a copy (not dup)
      * of union_comm or comp_comm
      */
-    if(my_comm == union_comm){
-      my_comms_.push_back(tmp_union_comm);
-    }
-    else if(my_comm == comp_comm){
-      my_comms_.push_back(tmp_comp_comm);
-    }
-    else if(my_comm == io_comm){
+    if(my_comm == io_comm){
       my_comms_.push_back(tmp_io_comm);
     }
     else{
-      assert(0);
+      my_comms_.push_back(MPI_COMM_NULL);
     }
 
-    tmp_comm = MPI_COMM_NULL;
-    if(node_comm != MPI_COMM_NULL){
-      ret = MPI_Comm_dup(node_comm, &tmp_comm); assert(ret == MPI_SUCCESS);
-    }
-    node_comms_.push_back(tmp_comm);
+    /* FIXME: To start using node_comm we need to create a node local
+     * comm with just the I/O procs and dup it
+     */
+    node_comms_.push_back(MPI_COMM_NULL);
   }
 }
 
@@ -147,11 +130,11 @@ SPIO_Util::TComm_info::~TComm_info()
 #endif
   assert(static_cast<int>(union_comms_.size()) == nthreads);
   for(int tidx = 1; tidx < nthreads; tidx++){
-    if(union_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&union_comms_[tidx]); }
+    assert(union_comms_[tidx] == MPI_COMM_NULL);
     if(io_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&io_comms_[tidx]); }
-    if(comp_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&comp_comms_[tidx]); }
-    if(intercomms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&intercomms_[tidx]); }
-    if(node_comms_[tidx] != MPI_COMM_NULL) { MPI_Comm_free(&node_comms_[tidx]); }
+    assert(comp_comms_[tidx] == MPI_COMM_NULL);
+    assert(intercomms_[tidx] == MPI_COMM_NULL);
+    assert(node_comms_[tidx] == MPI_COMM_NULL);
   }
 
   std::for_each(comm_infos_.begin(), comm_infos_.end(), [](MPI_Info &info) { MPI_Info_free(&info); });

From 6ab9044ad6f773ddd72b9db22f87bbb73b7a831b Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Fri, 5 Jun 2026 10:49:59 -0500
Subject: [PATCH 06/16] Code reformatting pio_err()

Re-formatting code in pio_err function

This commit does not change the functionality (or add new code)
---
 src/clib/core/pioc_support.cpp | 105 ++++++++++++++++-----------------
 1 file changed, 51 insertions(+), 54 deletions(-)

diff --git a/src/clib/core/pioc_support.cpp b/src/clib/core/pioc_support.cpp
index 3d43f8865a..607724d50f 100644
--- a/src/clib/core/pioc_support.cpp
+++ b/src/clib/core/pioc_support.cpp
@@ -1381,73 +1381,70 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file,
             int err_num, const char *fname, int line,
             const char *uerr_msg_fmt, ...)
 {
-    char err_msg[PIO_MAX_NAME + 1];
-    int err_handler = default_error_handler; /* Default error handler. */
-    int ret;
+  char err_msg[PIO_MAX_NAME + 1];
+  int err_handler = default_error_handler; /* Default error handler. */
+  int ret;
 
-    /* User must provide this. */
-    pioassert(fname, "file name must be provided", __FILE__, __LINE__);
+  /* User must provide this. */
+  pioassert(fname, "file name must be provided", __FILE__, __LINE__);
 
-    /* No harm, no foul. */
-    if (err_num == PIO_NOERR)
-        return PIO_NOERR;
+  /* No harm, no foul. */
+  if(err_num == PIO_NOERR) { return PIO_NOERR; }
 
-    /* Get the error message. */
-    if ((ret = PIOc_strerror_impl(err_num, err_msg, PIO_MAX_NAME)))
-        return ret;
+  /* Get the error message. */
+  ret = PIOc_strerror_impl(err_num, err_msg, PIO_MAX_NAME);
+  if(ret != PIO_NOERR) { return ret; }
 
-    /* PIO_MAX_NAME may not be enough to store the entire user error
-     * message, especially when long filenames are provided by the
-     * user in the message
-     */
-    const int UERR_MSG_MAX_LEN = 8192;
-    char uerr_msg[UERR_MSG_MAX_LEN + 1];
-    va_list argp;
-    va_start(argp, uerr_msg_fmt);
-    vsnprintf(uerr_msg, UERR_MSG_MAX_LEN, uerr_msg_fmt, argp);
-    va_end(argp);
+  /* PIO_MAX_NAME may not be enough to store the entire user error
+   * message, especially when long filenames are provided by the
+   * user in the message
+   */
+  const int UERR_MSG_MAX_LEN = 8192;
+  char uerr_msg[UERR_MSG_MAX_LEN + 1];
+  va_list argp;
 
-    /* If logging is in use, log an error message. */
-    LOG((1, "ERROR: %s. %s err_num = %d fname = %s line = %d", uerr_msg, err_msg, err_num, fname ? fname : "\0", line));
+  va_start(argp, uerr_msg_fmt);
+  vsnprintf(uerr_msg, UERR_MSG_MAX_LEN, uerr_msg_fmt, argp);
+  va_end(argp);
 
-    /* What error handler should we use? */
-    if (file)
-    {
-        ios = file->iosystem;
-        err_handler = ios->error_handler;
-    }
-    else if (ios)
-        err_handler = ios->error_handler;
+  /* If logging is in use, log an error message. */
+  LOG((1, "ERROR: %s. %s err_num = %d fname = %s line = %d", uerr_msg, err_msg, err_num, fname ? fname : "\0", line));
+
+  /* What error handler should we use? */
+  if(file){
+    ios = file->iosystem;
+    err_handler = ios->error_handler;
+  }
+  else if(ios){
+    err_handler = ios->error_handler;
+  }
 
-    LOG((2, "pio_err chose error handler = %d", err_handler));
+  LOG((2, "pio_err chose error handler = %d", err_handler));
 
-    /* Should we abort? */
-    if (err_handler == PIO_INTERNAL_ERROR)
-    {
-        /* For debugging only, this will print a traceback of the call tree.  */
-        piodie(fname, line, "An error occured, %s. %s (err=%d). Aborting since the error handler was set to PIO_INTERNAL_ERROR...", uerr_msg, err_msg, err_num);
-    }
-    else if (err_handler != PIO_RETURN_ERROR)
-    {
-        /* If the user does not explicitly ask to return error, print
-         * the error message in stderr on the root IO proc
-         */
-        bool print_err_msg = (ios) ? (ios->tcomm_info->get_union_comm_rank() == ios->tcomm_info->get_union_comm_io_root()) : true;
+  /* Should we abort? */
+  if(err_handler == PIO_INTERNAL_ERROR){
+    /* For debugging only, this will print a traceback of the call tree.  */
+    piodie(fname, line, "An error occured, %s. %s (err=%d). Aborting since the error handler was set to PIO_INTERNAL_ERROR...", uerr_msg, err_msg, err_num);
+  }
+  else if(err_handler != PIO_RETURN_ERROR){
+    /* If the user does not explicitly ask to return error, print
+     * the error message in stderr on the root IO proc
+     */
+    bool print_err_msg = (ios) ? (ios->tcomm_info->get_union_comm_rank() == ios->tcomm_info->get_union_comm_io_root()) : true;
 
-        if (print_err_msg)
-        {
-            fprintf(stderr, "PIO: ERROR: %s. %s (error num=%d), (%s:%d)\n", uerr_msg, err_msg, err_num, (fname) ? fname : "\0", line);
+    if(print_err_msg){
+      fprintf(stderr, "PIO: ERROR: %s. %s (error num=%d), (%s:%d)\n", uerr_msg, err_msg, err_num, (fname) ? fname : "\0", line);
 #ifdef _HDF5
-            H5Eprint2(H5E_DEFAULT, stderr);
+      H5Eprint2(H5E_DEFAULT, stderr);
 #endif
-            fflush(stderr);
-        }
+      fflush(stderr);
     }
+  }
 
-    /* For PIO_BCAST_ERROR and PIO_RETURN_ERROR error handlers
-     * just return the error code back to the caller
-     */
-    return err_num;
+  /* For PIO_BCAST_ERROR and PIO_RETURN_ERROR error handlers
+   * just return the error code back to the caller
+   */
+  return err_num;
 }
 
 /**

From 20209c2e591092b259f7779c209f049af49a5f58 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Fri, 5 Jun 2026 13:03:03 -0500
Subject: [PATCH 07/16] Updating err print logic for NULL global comms

Since threads no longer have valid global comms, updating the
logic to print error by checking for validity of the comms before
using it
---
 src/clib/core/pioc_support.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/clib/core/pioc_support.cpp b/src/clib/core/pioc_support.cpp
index 607724d50f..b36ec7e3b9 100644
--- a/src/clib/core/pioc_support.cpp
+++ b/src/clib/core/pioc_support.cpp
@@ -1430,7 +1430,15 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file,
     /* If the user does not explicitly ask to return error, print
      * the error message in stderr on the root IO proc
      */
-    bool print_err_msg = (ios) ? (ios->tcomm_info->get_union_comm_rank() == ios->tcomm_info->get_union_comm_io_root()) : true;
+    bool print_err_msg = true;
+    if(ios){
+      if(ios->tcomm_info->get_union_comm() != MPI_COMM_NULL){
+        print_err_msg = (ios->tcomm_info->get_union_comm_rank() == ios->tcomm_info->get_union_comm_io_root()) ? true : false;
+      }
+      else if(ios->tcomm_info->get_io_comm() != MPI_COMM_NULL){
+        print_err_msg = (ios->tcomm_info->get_io_comm_rank() == 0) ? true : false;
+      }
+    }
 
     if(print_err_msg){
       fprintf(stderr, "PIO: ERROR: %s. %s (error num=%d), (%s:%d)\n", uerr_msg, err_msg, err_num, (fname) ? fname : "\0", line);

From ca7317c56cc66ef0079782c8e98838a1fce586f3 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Fri, 5 Jun 2026 13:05:07 -0500
Subject: [PATCH 08/16] Fix barrier on global comm in thread

Threads should not be using global comms. Updating barrier in async
op to be on thread-specific I/O comm
---
 src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp b/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
index eb5949f328..704507067f 100644
--- a/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
+++ b/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
@@ -29,6 +29,7 @@ extern "C"{
 #include "spio_dbg_utils.hpp"
 #include "spio_dt_converter.hpp"
 #include "spio_hdf5_utils.hpp"
+#include "spio_async_tcomm.hpp"
 
 struct Hdf5_create_info{
   file_desc_t *file;
@@ -802,7 +803,7 @@ int pio_iosys_async_op_hdf5_write(void *pdata)
   }
 
   /* FIXME: Is this barrier needed ? */
-  MPI_Barrier(ios->io_comm);
+  MPI_Barrier(ios->tcomm_info->get_io_comm());
 
   iodesc->nasync_pend_ops--;
   file->npend_ops--;

From 90f0cb7de64d700a7ab17703d34bb15328c68190 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Sat, 6 Jun 2026 11:44:10 -0500
Subject: [PATCH 09/16] When closing file check thread-specific comm

When closing file ensure that we don't sync on the async I/O
threads
---
 src/clib/core/pio_file.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/clib/core/pio_file.cpp b/src/clib/core/pio_file.cpp
index a0f44564d9..a3b39bc25d 100644
--- a/src/clib/core/pio_file.cpp
+++ b/src/clib/core/pio_file.cpp
@@ -557,7 +557,11 @@ int spio_hard_closefile(iosystem_desc_t *ios, file_desc_t *file,
 
     if(sync_with_ioprocs){
       //if(ios->ioproc) { while(file->npend_ops){} }
-      MPI_Barrier(ios->union_comm);
+      /* For I/O async threads the union comm is NULL comm */
+      MPI_Comm sync_comm = ios->tcomm_info->get_union_comm();
+      if(sync_comm != MPI_COMM_NULL){
+        MPI_Barrier(sync_comm);
+      }
     }
 
     if(file->is_hard_closed) { return PIO_NOERR; }

From cf7c02576035dddd75205752d8cb2e7039bb212e Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Sat, 6 Jun 2026 11:46:07 -0500
Subject: [PATCH 10/16] When closing file use return error handler

Since not all error handlers are multi-threaded use the return
error handler when closing files (in async+threaded case).
---
 src/clib/core/pio_file.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/clib/core/pio_file.cpp b/src/clib/core/pio_file.cpp
index a3b39bc25d..766ddcf816 100644
--- a/src/clib/core/pio_file.cpp
+++ b/src/clib/core/pio_file.cpp
@@ -863,7 +863,12 @@ int spio_hard_closefile(iosystem_desc_t *ios, file_desc_t *file,
         }
     }
 
+#if PIO_USE_ASYNC_WR_THREAD
+    /* FIXME: Use the iosystem/file error handlers once error handlers are multi-threaded */
+    ierr = spio_handle_err(NULL, file, PIO_RETURN_ERROR, ierr, __FILE__, __LINE__);
+#else
     ierr = check_netcdf(NULL, file, ierr, __FILE__, __LINE__);
+#endif
     if(ierr != PIO_NOERR){
         LOG((1, "nc*_close failed, ierr = %d", ierr));
         return pio_err(NULL, file, ierr, __FILE__, __LINE__,

From ef48f4de0fcea11fe0b1634dc533394532cc87eb Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Sat, 6 Jun 2026 12:54:27 -0500
Subject: [PATCH 11/16] Test creates mahy decomps and files

The new test case creates many files and writes to them using one
decomp per file. The test case has two phases for each I/O type,

1) Create decomposition, create file and write to file
2) Sync data (close+open for sync if needed), read and verify,
   close+delete files, free decomposition

Phase 1 is executed for all files before moving to phase 2
---
 tests/general/pio_decomp_tests.F90.in | 107 ++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/tests/general/pio_decomp_tests.F90.in b/tests/general/pio_decomp_tests.F90.in
index af6a4fa0b4..ca43bf728c 100644
--- a/tests/general/pio_decomp_tests.F90.in
+++ b/tests/general/pio_decomp_tests.F90.in
@@ -537,3 +537,110 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_reuse_decomp
 
   call PIO_freedecomp(pio_tf_iosystem_, iodesc)
 PIO_TF_AUTO_TEST_SUB_END nc_write_1d_reuse_decomp
+
+! Create many decompositions and write to multiple files
+! (1 decomp per file, many vars per file, writes using the same buffer)
+! Note: For async+threaded case, the creation of the decompositions can
+! run in parallel to the writes to files
+PIO_TF_AUTO_TEST_SUB_BEGIN many_decomps_to_files
+  implicit none
+  integer, parameter :: NUM_FILES = 10
+  integer, parameter :: NUM_VARS = 20
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  integer, dimension(VEC_LOCAL_SZ) :: buf, rbuf
+  integer, dimension(1) :: dims
+  character(len=PIO_TF_MAX_STR_LEN) :: tmp_idx_str
+  character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var1'
+  character(len=*), parameter :: PIO_DIM_NAME = 'PIO_TF_test_var_dim1'
+
+  type :: test_file_info
+    character(len=PIO_TF_MAX_STR_LEN) :: filename
+    type(file_desc_t) :: pio_file
+    character(len=PIO_TF_MAX_STR_LEN) :: vnames(NUM_VARS)
+    type(var_desc_t)  :: pio_var(NUM_VARS)
+    integer :: pio_dim
+    type(io_desc_t) :: iodesc
+  end type test_file_info
+
+  type(test_file_info) :: tf_infos(NUM_FILES)
+
+  integer :: i, j, k, ierr
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    do j=1,NUM_FILES
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_int, dims, compdof, tf_infos(j)%iodesc)
+
+      write(tmp_idx_str, "(I0)") j
+      tf_infos(j)%filename = "many_decomps_to_files_test_" // trim(tmp_idx_str) // ".testfile"
+      ierr = PIO_createfile(pio_tf_iosystem_, tf_infos(j)%pio_file, iotypes(i), tf_infos(j)%filename, PIO_CLOBBER) 
+      PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(tf_infos(j)%filename))
+
+      ierr = PIO_def_dim(tf_infos(j)%pio_file, PIO_DIM_NAME, dims(1), tf_infos(j)%pio_dim)
+      PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(tf_infos(j)%filename))
+
+      do k=1,NUM_VARS
+        write(tmp_idx_str, "(I0)") k
+        tf_infos(j)%vnames(k) = PIO_VAR_NAME // trim(tmp_idx_str)
+        ierr = PIO_def_var(tf_infos(j)%pio_file, tf_infos(j)%vnames(k), PIO_int, (/tf_infos(j)%pio_dim/), tf_infos(j)%pio_var(k))
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(tf_infos(j)%filename))
+      end do
+
+      ierr = PIO_enddef(tf_infos(j)%pio_file)
+      PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(tf_infos(j)%filename))
+
+      do k=1,NUM_VARS
+        ! Write the variable out
+        call PIO_write_darray(tf_infos(j)%pio_file, tf_infos(j)%pio_var(k), tf_infos(j)%iodesc, buf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(tf_infos(j)%filename))
+      end do
+
+    end do
+
+    do j=1,NUM_FILES
+      ! For async+threaded, the re-open should sync the file
+#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
+      call PIO_closefile(tf_infos(j)%pio_file)
+
+      ierr = PIO_openfile(pio_tf_iosystem_, tf_infos(j)%pio_file, iotypes(i), tf_infos(j)%filename, PIO_nowrite)
+      PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(tf_infos(j)%filename))
+
+      do k=1,NUM_VARS
+        ierr = PIO_inq_varid(tf_infos(j)%pio_file, tf_infos(j)%vnames(k), tf_infos(j)%pio_var(k))
+        PIO_TF_CHECK_ERR(ierr, "Could not inq var1 file2 :" // trim(tf_infos(j)%filename))
+      end do
+#else
+      call PIO_syncfile(tf_infos(j)%pio_file)
+#endif
+      do k=1,NUM_VARS
+        rbuf = 0
+        call PIO_read_darray(tf_infos(j)%pio_file, tf_infos(j)%pio_var(k), tf_infos(j)%iodesc, rbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(tf_infos(j)%filename))
+
+        PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+      end do
+
+      call PIO_closefile(tf_infos(j)%pio_file)
+      call PIO_deletefile(pio_tf_iosystem_, tf_infos(j)%filename)
+      call PIO_freedecomp(pio_tf_iosystem_, tf_infos(j)%iodesc)
+    end do
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END many_decomps_to_files

From facf7f05690225a95312a96c95863200f28380e1 Mon Sep 17 00:00:00 2001
From: Jayesh Krishna <jayesh@mcs.anl.gov>
Date: Sat, 6 Jun 2026 18:11:49 -0700
Subject: [PATCH 12/16] Adding a simple stdout logger

Adding a simple stdout logger for tracing the async I/O ops (logs
are commented/disabled by default)

Once the tracer/logger is multi-threaded we can get rid of these
loggers
---
 .../core/iolib/hdf5/spio_async_hdf5_utils.cpp     | 15 +++++++++++++++
 src/clib/core/pio_file.cpp                        |  3 +++
 src/clib/util/spio_dbg_utils.hpp                  |  9 +++++++++
 3 files changed, 27 insertions(+)

diff --git a/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp b/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
index 704507067f..6b6e54ca48 100644
--- a/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
+++ b/src/clib/core/iolib/hdf5/spio_async_hdf5_utils.cpp
@@ -159,6 +159,7 @@ static inline void update_reg_infos_start_frame(std::vector<Util::RInfo> &reg_in
 
 void spio_iosys_async_op_hdf5_create_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     delete(static_cast<Hdf5_create_info *>(pdata));
   }
@@ -171,6 +172,7 @@ int spio_iosys_async_op_hdf5_create(void *pdata)
   Hdf5_create_info *cinfo = static_cast<Hdf5_create_info *>(pdata);
 
   assert(cinfo && cinfo->file && cinfo->file->iosystem);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + cinfo->fname);
 
   ret = spio_hdf5_create(cinfo->file->iosystem, cinfo->file, cinfo->fname.c_str());
 
@@ -218,6 +220,7 @@ int spio_iosys_async_hdf5_create_op_add(file_desc_t *file, const char *filename)
 
 void spio_iosys_async_op_hdf5_def_var_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     delete(static_cast<Hdf5_def_var_info *>(pdata));
   }
@@ -228,6 +231,7 @@ int spio_iosys_async_op_hdf5_def_var(void *pdata)
   int ret = PIO_NOERR;
 
   Hdf5_def_var_info *def_var_info = static_cast<Hdf5_def_var_info *>(pdata);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ": fname = " + def_var_info->file->fname + ", vname: " + def_var_info->vname);
 
   assert(def_var_info && def_var_info->file && def_var_info->file->iosystem);
 
@@ -281,6 +285,7 @@ int spio_iosys_async_hdf5_def_var_op_add(file_desc_t *file, const char *name,
 
 void spio_iosys_async_op_hdf5_put_att_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     Hdf5_put_att_info *info = static_cast<Hdf5_put_att_info *>(pdata);
     free(info->abuf);
@@ -297,6 +302,7 @@ int spio_iosys_async_op_hdf5_put_att(void *pdata)
   assert(info && info->file && info->file->iosystem);
   assert((info->varid >= 0) || (info->varid == PIO_GLOBAL));
   assert(info->alen >= 0);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ": fname = " + info->file->fname + ", aname: " + info->aname);
 
   ret = spio_hdf5_put_att(info->file->iosystem, info->file,
           info->varid, info->aname.c_str(), info->atype,
@@ -356,6 +362,7 @@ int spio_iosys_async_hdf5_put_att_op_add(file_desc_t *file, int varid,
 
 void spio_iosys_async_op_hdf5_enddef_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     delete(static_cast<Hdf5_enddef_info *>(pdata));
   }
@@ -368,6 +375,7 @@ int spio_iosys_async_op_hdf5_enddef(void *pdata)
   Hdf5_enddef_info *info = static_cast<Hdf5_enddef_info *>(pdata);
 
   assert(info && info->file && info->file->iosystem);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ": fname = " + info->file->fname);
 
   ret = spio_hdf5_enddef(info->file->iosystem, info->file);
 
@@ -415,6 +423,7 @@ int spio_iosys_async_hdf5_enddef_op_add(file_desc_t *file)
 
 void spio_iosys_async_op_hdf5_put_var_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     Hdf5_put_var_info *info = static_cast<Hdf5_put_var_info *>(pdata);
     free(info->vbuf);
@@ -430,6 +439,7 @@ int spio_iosys_async_op_hdf5_put_var(void *pdata)
 
   assert(info && info->file && info->file->iosystem);
   assert((info->varid >= 0) || (info->varid == PIO_GLOBAL));
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ": fname = " + info->file->fname + ", varid=" + std::to_string(info->varid));
 
   ret = spio_hdf5_put_var(info->file->iosystem, info->file,
           info->varid,
@@ -511,6 +521,7 @@ int spio_iosys_async_hdf5_put_var_op_add(file_desc_t *file, int varid,
 
 void spio_iosys_async_op_hdf5_set_frame_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
   if(pdata){
     delete(static_cast<Hdf5_set_frame_info *>(pdata));
   }
@@ -523,6 +534,7 @@ int spio_iosys_async_op_hdf5_set_frame(void *pdata)
   Hdf5_set_frame_info *info = static_cast<Hdf5_set_frame_info *>(pdata);
 
   assert(info && info->file && info->file->iosystem);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ": fname = " + info->file->fname + ", frame=" + std::to_string(info->frame));
 
   ret = spio_hdf5_set_frame(info->file, info->varid, info->frame);
 
@@ -572,6 +584,7 @@ int spio_iosys_async_hdf5_set_frame_op_add(file_desc_t *file, int varid, int fra
 
 int spio_wait_all_hdf5_async_ops(int iosysid)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ", iosysid=" + std::to_string(iosysid));
   unsigned long long int sleep_time = 0;
   /* Sleep for 0.5 seconds */
   const int SLEEP_TIME_IN_MILLISECONDS = 500;
@@ -597,6 +610,7 @@ int pio_iosys_async_op_hdf5_write(void *pdata)
   int ret = PIO_NOERR;
   Hdf5_wcache *wcache = static_cast<struct Hdf5_wcache *>(pdata);
   assert(wcache);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + ", fname=" + wcache->file->fname + ", nvars=" + std::to_string(wcache->nvars));
 
   file_desc_t *file = wcache->file;
   int nvars = wcache->nvars;
@@ -818,6 +832,7 @@ int pio_iosys_async_op_hdf5_write(void *pdata)
 
 void pio_iosys_async_op_hdf5_write_free(void *pdata)
 {
+  //SPIO_Util::Dbg_Util::Stdout_logger(__func__);
 #ifdef _HDF5
   Hdf5_wcache *wcache = static_cast<struct Hdf5_wcache *>(pdata);
   assert(wcache);
diff --git a/src/clib/core/pio_file.cpp b/src/clib/core/pio_file.cpp
index 766ddcf816..294bb81bfc 100644
--- a/src/clib/core/pio_file.cpp
+++ b/src/clib/core/pio_file.cpp
@@ -16,6 +16,7 @@
 #include <functional>
 #include "spio_hdf5_utils.hpp"
 #include "spio_async_tcomm.hpp"
+#include "spio_dbg_utils.hpp"
 
 #ifdef _ADIOS2
 #include "../../tools/adios2pio-nm/adios2pio-nm-lib-c.h"
@@ -549,6 +550,7 @@ int spio_hard_closefile(iosystem_desc_t *ios, file_desc_t *file,
 #endif
 
     assert(ios && file);
+    //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + file->fname);
 
     /* Get the lock before proceeding */
     std::lock_guard<std::mutex> lg(*(file->pmtx));
@@ -928,6 +930,7 @@ int spio_hard_closefile(iosystem_desc_t *ios, file_desc_t *file,
 int spio_soft_closefile(iosystem_desc_t *ios, file_desc_t *file)
 {
   assert(ios && file && ios->ioproc);
+  //SPIO_Util::Dbg_Util::Stdout_logger(std::string(__func__) + file->fname);
 
 #if PIO_USE_ASYNC_WR_THREAD
   return pio_iosys_async_file_close_op_add(file);
diff --git a/src/clib/util/spio_dbg_utils.hpp b/src/clib/util/spio_dbg_utils.hpp
index 749afb120d..46f03c6a53 100644
--- a/src/clib/util/spio_dbg_utils.hpp
+++ b/src/clib/util/spio_dbg_utils.hpp
@@ -64,6 +64,15 @@ namespace SPIO_Util{
       return ostr.str();
     }
 
+    class Stdout_logger{
+      public:
+        Stdout_logger(const std::string &info) : info_(info) { std::cout << "Entering " << info_.c_str() << "\n" << std::flush; }
+        ~Stdout_logger() { std::cout << "Leaving " << info_.c_str() << "\n" << std::flush; }
+
+      private:
+        const std::string info_;
+    };
+
     std::string get_iodesc_info(io_desc_t *ios);
 
     void get_stack_trace(std::vector<std::string> &st);

From 17de6869a21a7d74445c5ef85201e26b035437ea Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Sun, 7 Jun 2026 00:36:13 -0500
Subject: [PATCH 13/16] Adding config opt to trace I/O decomps

Adding a new configure option to choose whether to trace I/O
decompositions (while tracing APIs). By default its turned off
---
 CMakeLists.txt                       |  1 +
 src/clib/CMakeLists.txt              |  8 ++++++++
 src/clib/api/spio_io_decomp_api.cpp  |  4 ++++
 src/clib/pio_config.h.in             |  5 +++++
 src/clib/util/spio_tracer_decomp.cpp | 18 +++++++++---------
 5 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 056acea84c..b595e058d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,6 +37,7 @@ option (PIO_ENABLE_INTERNAL_TIMING    "Gather and print GPTL timing stats"  OFF)
 option (PIO_ENABLE_API_TIMING    "Enable timer for all (costly) SCORPIO APIs (requires PIO_ENABLE_TIMING=ON)"          ON)
 option (PIO_ENABLE_API_TRACING    "Enable tracing of SCORPIO APIs"          OFF)
 option (PIO_ENABLE_API_VAR_TRACING    "Enable tracing of variables when tracing SCORPIO APIs" OFF)
+option (PIO_ENABLE_API_DECOMP_TRACING    "Enable tracing of I/O decompositions when tracing SCORPIO APIs" OFF)
 option (PIO_ENABLE_IO_STATS  "Gather and print I/O performance stats"       ON)
 option (PIO_ENABLE_LOGGING   "Enable debug logging (large output possible)" OFF)
 option (PIO_ENABLE_DOC       "Enable building SCORPIO documentation"        ON)
diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt
index ae4ab27703..42470b27c2 100644
--- a/src/clib/CMakeLists.txt
+++ b/src/clib/CMakeLists.txt
@@ -438,10 +438,18 @@ if (PIO_ENABLE_API_TRACING)
     message(STATUS "Disabling variable tracing in APIs (default)")
     set(ENABLE_API_VAR_TRACING 0)
   endif ()
+  if (PIO_ENABLE_API_DECOMP_TRACING)
+    message(STATUS "Enabling I/O decomposition tracing in APIs")
+    set(ENABLE_API_DECOMP_TRACING 1)
+  else ()
+    message(STATUS "Disabling I/O decomposition tracing in APIs (default)")
+    set(ENABLE_API_DECOMP_TRACING 0)
+  endif ()
 else ()
   message(STATUS "API tracing disabled (default)")
   set(ENABLE_API_TRACING 0)
   set(ENABLE_API_VAR_TRACING 0)
+  set(ENABLE_API_DECOMP_TRACING 0)
 endif ()
 
 # The MPI library detection was done in the top level
diff --git a/src/clib/api/spio_io_decomp_api.cpp b/src/clib/api/spio_io_decomp_api.cpp
index ac1046300d..3a914b86d8 100644
--- a/src/clib/api/spio_io_decomp_api.cpp
+++ b/src/clib/api/spio_io_decomp_api.cpp
@@ -28,7 +28,9 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
                               compmap, ioidp, rearr, iostart, iocount);
 
 #if SPIO_ENABLE_API_TRACING
+#if SPIO_ENABLE_API_DECOMP_TRACING
   tr.set_decomp_info((ioidp) ? (*ioidp) : -1, compmap, maplen);
+#endif
   tr.add_rval("*ioidp", (ioidp) ? (*ioidp) : -1);
 #endif
   return ret;
@@ -80,7 +82,9 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
                                 compmap, ioidp, rearranger, iostart, iocount);
 
 #if SPIO_ENABLE_API_TRACING
+#if SPIO_ENABLE_API_DECOMP_TRACING
   tr.set_decomp_info((ioidp) ? (*ioidp) : -1, compmap, maplen);
+#endif
   tr.add_rval("*ioidp", (ioidp) ? (*ioidp) : -1);
 #endif
   return ret;
diff --git a/src/clib/pio_config.h.in b/src/clib/pio_config.h.in
index 43105607b4..928f96566d 100644
--- a/src/clib/pio_config.h.in
+++ b/src/clib/pio_config.h.in
@@ -183,6 +183,11 @@
  *  0 otherwise */
 #define SPIO_ENABLE_API_VAR_TRACING @ENABLE_API_VAR_TRACING@
 
+/** Set to 1 if the library is configured to trace API calls &
+ *  trace I/O decomposition data in the API calls,
+ *  0 otherwise */
+#define SPIO_ENABLE_API_DECOMP_TRACING @ENABLE_API_DECOMP_TRACING@
+
 /** Set to 1 if the library is configured to use PnetCDF independent data mode,
  *  0 otherwise */
 #define PIO_USE_INDEP_MODE @USE_INDEP_MODE@
diff --git a/src/clib/util/spio_tracer_decomp.cpp b/src/clib/util/spio_tracer_decomp.cpp
index 1baa837ab1..6655b9c759 100644
--- a/src/clib/util/spio_tracer_decomp.cpp
+++ b/src/clib/util/spio_tracer_decomp.cpp
@@ -16,7 +16,7 @@
 #include "pio.h"
 #include "pio_internal.h"
 #include "mpi.h"
-#if PIO_USE_PNETCDF
+#if SPIO_ENABLE_API_DECOMP_TRACING
   #include "pnetcdf.h"
 #endif
 #include "spio_tracer.hpp"
@@ -120,7 +120,7 @@ void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::init(void )
 
   fname_ = get_trace_decomp_tmp_fname(iosysid_, comm_, comm_rank_, wrank);
 
-#if PIO_USE_PNETCDF
+#if SPIO_ENABLE_API_DECOMP_TRACING
   int flags = NC_CLOBBER | NC_64BIT_DATA;
   ret = ncmpi_create(comm_, fname_.c_str(), flags, info, &ncid_);
   if(ret != NC_NOERR){
@@ -171,7 +171,7 @@ void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::init(void )
     throw std::runtime_error(std::string("Ending NetCDF define mode in I/O decomposition log file, file = ") + fname_ +
             std::string(", failed. iosysid = ") + std::to_string(iosysid_));
   }
-#endif // PIO_USE_PNETCDF
+#endif // SPIO_ENABLE_API_DECOMP_TRACING
 
   ret = MPI_Info_free(&info);
   assert(ret == MPI_SUCCESS);
@@ -249,7 +249,7 @@ void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::log_decomp(int ioid, con
   std::string decomp_vname = std::string("decomp_") + std::to_string(ioid);
   int decomp_varid = INVALID_NCID;
 
-#if PIO_USE_PNETCDF
+#if SPIO_ENABLE_API_DECOMP_TRACING
   ret = ncmpi_redef(ncid_);
   if(ret != NC_NOERR){
     throw std::runtime_error(std::string("Unable to enter redef mode to write decompositing map, ncid = ") + std::to_string(ncid_) +
@@ -290,12 +290,12 @@ void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::log_decomp(int ioid, con
     throw std::runtime_error(std::string("Error while waiting after writing the decomposition map, ncid = ") + std::to_string(ncid_) +
             std::string(", vname = ") + decomp_vname);
   }
-#endif // PIO_USE_PNETCDF
+#endif // SPIO_ENABLE_API_DECOMP_TRACING
 }
 
 void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::finalize(void )
 {
-#if PIO_USE_PNETCDF
+#if SPIO_ENABLE_API_DECOMP_TRACING
   if(ncid_ != INVALID_NCID){
     int ret = MPI_SUCCESS;
     int dim_comm_sz_dimid = INVALID_NCID, dim_ndecompsx3_dimid = INVALID_NCID;
@@ -367,14 +367,14 @@ void SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::finalize(void )
 
     ncid_ = INVALID_NCID;
   }
-#endif // PIO_USE_PNETCDF
+#endif // SPIO_ENABLE_API_DECOMP_TRACING
 }
 
 int SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::get_dim_id(PIO_Offset dimsz)
 {
   int dimid = INVALID_NCID, ret = 0;
 
-#if PIO_USE_PNETCDF
+#if SPIO_ENABLE_API_DECOMP_TRACING
   std::map<PIO_Offset, int>::iterator dim_id_iter = dim_sz_to_id_.find(dimsz);
   if(dim_id_iter == dim_sz_to_id_.end()){
     assert(ncid_ != INVALID_NCID);
@@ -405,7 +405,7 @@ int SPIO_Util::Tracer::Decomp_Utils::Decomp_nc_logger::get_dim_id(PIO_Offset dim
   else{
     dimid = dim_id_iter->second;
   }
-#endif // PIO_USE_PNETCDF
+#endif // SPIO_ENABLE_API_DECOMP_TRACING
   return dimid;
 }
 

From 89dd7dce25565794b566eaa9ed2f52355554cf85 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Tue, 30 Jun 2026 14:39:49 -0500
Subject: [PATCH 14/16] Wait on writes before read

Since not all MPI libraries support concurrent operations (from
multiple threads) on multiple files ensure that we wait on
pending async ops before proceeding with a synchronous operation
(read of a different file, sync op using a different iotype etc)

This is not efficient and we need to improve read perf in a
future PR.
---
 src/clib/core/pioc_support.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/clib/core/pioc_support.cpp b/src/clib/core/pioc_support.cpp
index b36ec7e3b9..c63cc1ed42 100644
--- a/src/clib/core/pioc_support.cpp
+++ b/src/clib/core/pioc_support.cpp
@@ -2858,7 +2858,6 @@ int spio_createfile_int(int iosysid, int *ncidp, const int *iotype, const char *
 
 #if PIO_USE_ASYNC_WR_THREAD
     /* FIXME: Relax this wait */
-    /*
     if((*iotype != PIO_IOTYPE_HDF5) && (*iotype != PIO_IOTYPE_HDF5C)){
       ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
       if(ierr != PIO_NOERR){
@@ -2866,7 +2865,6 @@ int spio_createfile_int(int iosysid, int *ncidp, const int *iotype, const char *
                         "Creating file (%s) failed. Error waiting on all pending asynchronous HDF5 ops", filename);
       }
     }
-    */
 
     ierr = spio_close_soft_closed_file(filename);
     if(ierr != PIO_NOERR){
@@ -4591,6 +4589,11 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
   /* To wait on all hdf5 async ops (for debugging),
    * ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
    */
+  ierr = spio_wait_all_hdf5_async_ops(ios->iosysid);
+  if(ierr != PIO_NOERR){
+    return pio_err(ios, NULL, ierr, __FILE__, __LINE__,
+                    "Creating file (%s) failed. Error waiting on async ops", filename);
+  }
 #endif
 
   /* Allocate space for the file info. */

From 95739d99be3eb38c557913a311337b0505cac2db Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Sat, 11 Jul 2026 16:03:51 -0500
Subject: [PATCH 15/16] Adding missing static keyword for mtx

Adding missing static for function level mutex when creating
thread-specific comms.
---
 src/clib/core/progress_engine/spio_async_tcomm.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/clib/core/progress_engine/spio_async_tcomm.cpp b/src/clib/core/progress_engine/spio_async_tcomm.cpp
index 5f16592a0c..1f254b9c15 100644
--- a/src/clib/core/progress_engine/spio_async_tcomm.cpp
+++ b/src/clib/core/progress_engine/spio_async_tcomm.cpp
@@ -106,7 +106,7 @@ MPI_Info *SPIO_Util::TComm_info::create_mpi_info(void )
   int ret = MPI_SUCCESS;
   MPI_Info info;
 #if PIO_USE_ASYNC_WR_THREAD
-  std::mutex mtx;
+  static std::mutex mtx;
   std::lock_guard<std::mutex> lg(mtx);
 #endif
 

From cdb4d19a062961ce94d8178e776622bac553ae60 Mon Sep 17 00:00:00 2001
From: jayeshkrishna <jayesh@mcs.anl.gov>
Date: Fri, 21 Aug 2026 10:30:39 -0500
Subject: [PATCH 16/16] Test using many iodecomps with many iotypes

Create many decompositions and write to multiple files with many
I/O types

* One decomp per file (so that we create many iodecomps). All
  decomps are the same (logically)
* Many vars written per file (Writes using the same buffer)

This allows potential concurrent execution of I/O decomposition
calls with reading/writing data (when using async I/O)
---
 tests/general/pio_decomp_tests.F90.in | 149 ++++++++++++++++++++++++++
 1 file changed, 149 insertions(+)

diff --git a/tests/general/pio_decomp_tests.F90.in b/tests/general/pio_decomp_tests.F90.in
index ca43bf728c..a7ccaa72d9 100644
--- a/tests/general/pio_decomp_tests.F90.in
+++ b/tests/general/pio_decomp_tests.F90.in
@@ -644,3 +644,152 @@ PIO_TF_AUTO_TEST_SUB_BEGIN many_decomps_to_files
     deallocate(iotype_descs)
   end if
 PIO_TF_AUTO_TEST_SUB_END many_decomps_to_files
+
+! Create many decompositions and write to multiple files with many I/O types
+! (one decomp per file, many vars per file, writes using the same buffer)
+! If NUM_FILES_PER_IOTYPE = 2 and number of available I/O types, num_iotypes=2,
+! (assuming available I/O types are hdf5 and pnetcdf)
+! 1) Define+Write loop : Creates I/O decomps, files and defines+writes vars for each I/O type
+!     Create I/O decomp iodecomp1
+!     Create file many_decomps_many_iotypes_test_1.hdf5.testfile & define+write (using iodecomp1) vars
+!     Create I/O decomp iodecomp2
+!     Create file many_decomps_many_iotypes_test_2.pnetcdf.testfile & define+write (using iodecomp2) vars
+!     Create I/O decomp iodecomp3
+!     Create file many_decomps_many_iotypes_test_3.hdf5.testfile & define+write (using iodecomp3) vars
+!     Create I/O decomp iodecomp4
+!     Create file many_decomps_many_iotypes_test_4.pnetcdf.testfile & define+write (using iodecomp4) vars
+! 2) Sync loop : Sync variables to the created files
+!     Sync variables to many_decomps_many_iotypes_test_1.hdf5.testfile
+!     Sync variables to many_decomps_many_iotypes_test_2.pnetcdf.testfile
+!     Sync variables to many_decomps_many_iotypes_test_3.hdf5.testfile
+!     Sync variables to many_decomps_many_iotypes_test_4.pnetcdf.testfile
+! 3) Read+Finalize loop : Read contents of files and close/delete them in the order
+!     it was synced the "Sync loop"
+!
+! Note: For async+threaded case, the creation of the decompositions can
+! run in parallel to the writes to files. Also if one of the I/O types (e.g. hdf5)
+! is async while another is not these sequences mixes file operations between async
+! and non-async I/O types
+PIO_TF_AUTO_TEST_SUB_BEGIN many_decomps_many_iotypes
+  implicit none
+  integer, parameter :: NUM_FILES_PER_IOTYPE = 5
+  integer, parameter :: NUM_VARS = 20
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  integer, dimension(VEC_LOCAL_SZ) :: buf, rbuf
+  integer, dimension(1) :: dims
+  character(len=PIO_TF_MAX_STR_LEN) :: tmp_idx_str
+  character(len=*), parameter :: PIO_VAR_NAME = 'PIO_TF_test_var1'
+  character(len=*), parameter :: PIO_DIM_NAME = 'PIO_TF_test_var_dim1'
+
+  type :: test_file_info
+    character(len=PIO_TF_MAX_STR_LEN) :: filename
+    integer :: iotype
+    type(file_desc_t) :: pio_file
+    character(len=PIO_TF_MAX_STR_LEN) :: vnames(NUM_VARS)
+    type(var_desc_t)  :: pio_var(NUM_VARS)
+    integer :: pio_dim
+    type(io_desc_t) :: iodesc
+  end type test_file_info
+
+  type(test_file_info), dimension(:), allocatable :: tf_infos
+  integer  :: num_files = 0
+
+  integer :: i, j, k, l, m, ierr
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+
+  num_files = NUM_FILES_PER_IOTYPE * num_iotypes
+  allocate(tf_infos(num_files))
+
+  ! ============== DEFINE+WRITE LOOP =====================
+  do m=1,NUM_FILES_PER_IOTYPE
+    ! Create files, define vars and write in a loop for all I/O types
+    do l=1,num_iotypes
+      ! Index into the global file info array
+      j = (m - 1) * num_iotypes + l
+
+      ! One decomp per file
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_int, dims, compdof, tf_infos(j)%iodesc)
+
+      write(tmp_idx_str, "(I0)") j
+      tf_infos(j)%filename = "many_decomps_many_iotypes_test_" // trim(tmp_idx_str) // "." // trim(iotype_descs(l)) // ".testfile"
+      tf_infos(j)%iotype = iotypes(l)
+
+      ierr = PIO_createfile(pio_tf_iosystem_, tf_infos(j)%pio_file, tf_infos(j)%iotype, tf_infos(j)%filename, PIO_CLOBBER) 
+      PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(tf_infos(j)%filename))
+
+      ierr = PIO_def_dim(tf_infos(j)%pio_file, PIO_DIM_NAME, dims(1), tf_infos(j)%pio_dim)
+      PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(tf_infos(j)%filename))
+
+      do k=1,NUM_VARS
+        write(tmp_idx_str, "(I0)") k
+        tf_infos(j)%vnames(k) = PIO_VAR_NAME // trim(tmp_idx_str)
+        ierr = PIO_def_var(tf_infos(j)%pio_file, tf_infos(j)%vnames(k), PIO_int, (/tf_infos(j)%pio_dim/), tf_infos(j)%pio_var(k))
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(tf_infos(j)%filename))
+      end do
+
+      ierr = PIO_enddef(tf_infos(j)%pio_file)
+      PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(tf_infos(j)%filename))
+
+      ! Note : If writes are async, the writes below can overlap with PIO_initdecomp() calls in the subsequent loop
+      do k=1,NUM_VARS
+        ! Write the variable out
+        call PIO_write_darray(tf_infos(j)%pio_file, tf_infos(j)%pio_var(k), tf_infos(j)%iodesc, buf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(tf_infos(j)%filename))
+      end do
+    end do
+  end do
+
+  ! ============== SYNC LOOP =====================
+  do j=1,num_files
+    ! For async+threaded, the re-open should sync the file
+#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
+    call PIO_closefile(tf_infos(j)%pio_file)
+
+    ierr = PIO_openfile(pio_tf_iosystem_, tf_infos(j)%pio_file, tf_infos(j)%iotype, tf_infos(j)%filename, PIO_nowrite)
+    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(tf_infos(j)%filename))
+
+    do k=1,NUM_VARS
+      ierr = PIO_inq_varid(tf_infos(j)%pio_file, tf_infos(j)%vnames(k), tf_infos(j)%pio_var(k))
+      PIO_TF_CHECK_ERR(ierr, "Could not inq var1 file2 :" // trim(tf_infos(j)%filename))
+    end do
+#else
+    call PIO_syncfile(tf_infos(j)%pio_file)
+#endif
+  end do
+
+  ! ============== READ+FINALIZE LOOP =====================
+  do j=1,num_files
+    do k=1,NUM_VARS
+      rbuf = 0
+      call PIO_read_darray(tf_infos(j)%pio_file, tf_infos(j)%pio_var(k), tf_infos(j)%iodesc, rbuf, ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(tf_infos(j)%filename))
+
+      PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+    end do
+
+    call PIO_closefile(tf_infos(j)%pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, tf_infos(j)%filename)
+    call PIO_freedecomp(pio_tf_iosystem_, tf_infos(j)%iodesc)
+  end do
+
+  deallocate(tf_infos)
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END many_decomps_many_iotypes