Enable user to take ownership of MPI

AUTHORS.txt

@@ -44,6 +44,8 @@ Dr Ian Bush [consultant]
     HPC
 
 External contributors:
+Íñigo Aréjula Aísa
+    patched validation error in the experimental user-owned MPI interface (#722)
 Daniel Expósito Patiño
     patched the applyMultiStateControlledSqrtSwap C++ signature (#737)
 Diogo Pratas Maia
@@ -70,8 +72,8 @@ SchineCompton
     patched GPU Cmake Release build
 Christopher J. Anders
     patched Cmake build when multhithreading defaults off
-    revsied Cmake min version for GPU build
+    revised Cmake min version for GPU build
 Gleb Struchalin
     patched the cmake standalone build
 Milos Prokop
-    implemented serial prototype of initDiagonalOpFromPauliHamil
+    implemented serial prototype of initDiagonalOpFromPauliHamil

CMakeLists.txt

@@ -145,6 +145,12 @@ option(
 )
 message(STATUS "Distribution is turned ${ENABLE_DISTRIBUTION}. Set ENABLE_DISTRIBUTION to modify.")
 
+option(
+  ENABLE_SUBCOMM
+  "Whether QuEST will be built with support for restricting it to a user-defined MPI communicator. Turned OFF by default."
+  OFF
+)
+message(STATUS "Custom communicator support is turned ${ENABLE_SUBCOMM}. Set ENABLE_SUBCOMM to modify.")
 
 # GPU Acceleration
 option(
@@ -207,6 +213,9 @@ if (ENABLE_CUQUANTUM AND NOT ENABLE_CUDA)
   message(FATAL_ERROR "Use of cuQuantum requires CUDA.")
 endif()
 
+if (ENABLE_SUBCOMM AND NOT ENABLE_DISTRIBUTION)
+  message(FATAL_ERROR "Distribution must be enabled to make use of a user-defined communicator for QuEST.")
+endif()
 
 if(WIN32)
 
@@ -402,8 +411,11 @@ endif()
 # MPI
 if (ENABLE_DISTRIBUTION)
   find_package(MPI REQUIRED
+    # Component CXX is the C api usable from C++
+    # NOT the deprecated C++ API
     COMPONENTS CXX
   )
+
   target_link_libraries(QuEST
     PRIVATE
     MPI::MPI_CXX
@@ -476,6 +488,7 @@ endif()
 # set vars which will be written to config.h.in (auto-converted to 0 or 1)
 set(COMPILE_OPENMP ${ENABLE_MULTITHREADING})
 set(COMPILE_MPI ${ENABLE_DISTRIBUTION})
+set(COMPILE_SUBCOMM ${ENABLE_SUBCOMM})
 set(COMPILE_CUQUANTUM ${ENABLE_CUQUANTUM})
 set(INCLUDE_DEPRECATED_FUNCTIONS ${ENABLE_DEPRECATED_API})
 
@@ -530,6 +543,10 @@ add_executable(min_example
 )
 target_link_libraries(min_example PRIVATE QuEST::QuEST)
 
+if (ENABLE_DISTRIBUTION AND ENABLE_SUBCOMM)
+  target_link_libraries(min_example PRIVATE MPI::MPI_CXX)
+endif()
+
 if (INSTALL_BINARIES)
   install(TARGETS min_example
     RUNTIME

docs/cmake.md

@@ -39,6 +39,7 @@ make
 | `INSTALL_BINARIES` | (`OFF`), `ON` | Determines whether compiled binaries such as the examples will be installed as well as the QuEST library. |
 | `ENABLE_MULTITHREADING` | (`ON`), `OFF` | Determines whether QuEST will be built with support for parallelisation with OpenMP. |
 | `ENABLE_DISTRIBUTION` | (`OFF`), `ON` | Determines whether QuEST will be built with support for parallelisation with MPI. |
+| `ENABLE_SUBCOMM` | (`OFF`), `ON` | Determines whether QuEST will be built with support for custom MPI communicators. _**Note**: This has the unfortunate side-effect of requiring the MPI header in the public header for QuEST, meaning MPI will become a dependency of any application or library which includes the QuEST header whether it uses MPI or not._ |
 | `ENABLE_CUDA` | (`OFF`), `ON` | Determines whether QuEST will be built with support for NVIDIA GPU acceleration. If turned on, `CMAKE_CUDA_ARCHITECTURES` should probably also be set. |
 | `ENABLE_CUQUANTUM` | (`OFF`), `ON` | Determines whether QuEST will make use of the NVIDIA CuQuantum library. Cannot be turned on if `ENABLE_CUDA` is off. |
 | `ENABLE_HIP` | (`OFF`), `ON` | Determines whether QuEST will be built with support for AMD GPU acceleration. If turned on, `CMAKE_HIP_ARCHITECTURES` should probably also be set. |

examples/CMakeLists.txt

@@ -20,6 +20,10 @@ function(add_example direc in_fn)
   add_executable(${target} ${in_fn})
   target_link_libraries(${target} PUBLIC QuEST)
 
+  if (ENABLE_DISTRIBUTION AND ENABLE_SUBCOMM)
+    target_link_libraries(${target} PRIVATE MPI::MPI_CXX)
+  endif()
+
   if (INSTALL_BINARIES)
     install(
       TARGETS ${target}

quest/include/config.h.in

@@ -37,6 +37,7 @@
 #if defined(FLOAT_PRECISION)              || \
     defined(COMPILE_OPENMP)               || \
     defined(COMPILE_MPI)                  || \
+    defined(COMPILE_SUBCOMM)              || \
     defined(COMPILE_CUDA)                 || \
     defined(COMPILE_HIP)                  || \
     defined(COMPILE_CUQUANTUM)            || \
@@ -79,6 +80,7 @@
 // crucial to QuEST source (informs external library usage)
 #cmakedefine01 COMPILE_OPENMP
 #cmakedefine01 COMPILE_MPI
+#cmakedefine01 COMPILE_SUBCOMM
 #cmakedefine01 COMPILE_CUDA
 #cmakedefine01 COMPILE_CUQUANTUM
 
@@ -118,6 +120,7 @@
 #if ! defined(FLOAT_PRECISION)              || \
     ! defined(COMPILE_OPENMP)               || \
     ! defined(COMPILE_MPI)                  || \
+    ! defined(COMPILE_SUBCOMM)              || \
     ! defined(COMPILE_CUDA)                 || \
     ! defined(COMPILE_HIP)                  || \
     ! defined(COMPILE_CUQUANTUM)            || \
@@ -144,6 +147,7 @@
 
 #if ! (COMPILE_OPENMP               == 0 || COMPILE_OPENMP               == 1) || \
     ! (COMPILE_MPI                  == 0 || COMPILE_MPI                  == 1) || \
+    ! (COMPILE_SUBCOMM             == 0  || COMPILE_SUBCOMM              == 1) || \
     ! (COMPILE_CUDA                 == 0 || COMPILE_CUDA                 == 1) || \
     ! (COMPILE_HIP                  == 0 || COMPILE_HIP                  == 1) || \
     ! (COMPILE_CUQUANTUM            == 0 || COMPILE_CUQUANTUM            == 1) || \

quest/include/environment.h

@@ -14,6 +14,8 @@
 #ifndef ENVIRONMENT_H
 #define ENVIRONMENT_H
 
+#include <stdbool.h>
+
 // enable invocation by both C and C++ binaries
 #ifdef __cplusplus
 extern "C" {
@@ -36,6 +38,7 @@ typedef struct {
     int isMultithreaded;
     int isGpuAccelerated;
     int isDistributed;
+    bool userOwnsMpi;
 
     // deployment modes which cannot be directly changed after compilation
     int isCuQuantumEnabled;
@@ -61,6 +64,12 @@ void initQuESTEnv();
  */
 void initCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMultithread);
 
+/** @notyetdoced
+ *  Advanced initialiser which lets the user positively declare that they take responsibility for MPI.
+ *  This means we assume they have called MPI_Init, and that they will call MPI_Finalize.
+ */
+void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread);
+
 /// @notyetdoced
 void finalizeQuESTEnv();
 

quest/include/quest.h

@@ -45,6 +45,7 @@
 #include "quest/include/operations.h"
 #include "quest/include/paulis.h"
 #include "quest/include/qureg.h"
+#include "quest/include/subcommunicator.h"
 #include "quest/include/matrices.h"
 #include "quest/include/wrappers.h"
 

quest/include/subcommunicator.h

@@ -0,0 +1,31 @@
+#ifndef SUBCOMMUNICATOR_H
+#define SUBCOMMUNICATOR_H
+
+#include "quest/include/config.h" 
+
+#if COMPILE_MPI && COMPILE_SUBCOMM
+
+#include <mpi.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @notyetdoced
+ *  Advanced initialiser which allows the user to provide an MPI communicator for QuEST to use.
+ *  Use of this initialiser implies userOwnsMpi = true, (exposed by initCustomMpiQuESTEnv) and 
+ *  therefore that they have already initialised MPI, and they will call MPI_Finalize at the 
+ *  appropriate time.
+ *
+ *  The user-provided MPI communicator undergoes the same validation procedure as any that QuEST
+ *  would use, and so must contain a power-of-2 number of processes.
+ */
+void initCustomMpiCommQuESTEnv(MPI_Comm questComm, int useGpuAccel, int useMultithread);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+#endif

quest/src/api/CMakeLists.txt

@@ -12,6 +12,7 @@ target_sources(QuEST
   operations.cpp
   paulis.cpp
   qureg.cpp
+  subcommunicator.cpp
   trotterisation.cpp
   types.cpp
-)
+)

quest/src/api/environment.cpp

@@ -71,7 +71,7 @@ static bool hasEnvBeenFinalized = false;
  */
 
 
-void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMultithread, const char* caller) {
+void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller) {
 
     // ensure that we are never re-initialising QuEST (even after finalize) because
     // this leads to undefined behaviour in distributed mode, as per the MPI
@@ -93,8 +93,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti
     // and before any GPU initialisation and validation, since we will
     // perform that specifically upon the MPI-process-bound GPU(s). Further,
     // we can make sure validation errors are reported only by the root node.
-    if (useDistrib)
-        comm_init();
+    comm_init(useDistrib, userOwnsMpi);
 
     validate_newEnvDistributedBetweenPower2Nodes(caller);
 
@@ -145,6 +144,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti
     globalEnvPtr->isMultithreaded     = useMultithread;
     globalEnvPtr->isGpuAccelerated    = useGpuAccel;
     globalEnvPtr->isDistributed       = useDistrib;
+    globalEnvPtr->userOwnsMpi         = userOwnsMpi;
     globalEnvPtr->isCuQuantumEnabled  = useCuQuantum;
     globalEnvPtr->isGpuSharingEnabled = permitGpuSharing;
 
@@ -153,6 +153,11 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti
     globalEnvPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1;
 }
 
+void updateQuESTEnvDistInfo() {
+    globalEnvPtr->rank     = (globalEnvPtr->isDistributed)? comm_getRank()     : 0;
+    globalEnvPtr->numNodes = (globalEnvPtr->isDistributed)? comm_getNumNodes() : 1;
+    return;
+}
 
 
 /*
@@ -187,10 +192,11 @@ void printCompilationInfo() {
 
     print_table(
         "compilation", {
-        {"isMpiCompiled",       comm_isMpiCompiled()},
-        {"isGpuCompiled",       gpu_isGpuCompiled()},
-        {"isOmpCompiled",       cpu_isOpenmpCompiled()},
-        {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()},
+        {"isMpiCompiled",                comm_isMpiCompiled()},
+        {"isMpiSubCommunicatorCompiled", comm_isMpiSubCommunicatorCompiled()},
+        {"isGpuCompiled",                gpu_isGpuCompiled()},
+        {"isOmpCompiled",                cpu_isOpenmpCompiled()},
+        {"isCuQuantumCompiled",          gpu_isCuQuantumCompiled()},
     });
 }
 
@@ -200,6 +206,7 @@ void printDeploymentInfo() {
     print_table(
         "deployment", {
         {"isMpiEnabled",        globalEnvPtr->isDistributed},
+        {"doesUserOwnMpi",      globalEnvPtr->userOwnsMpi},
         {"isGpuEnabled",        globalEnvPtr->isGpuAccelerated},
         {"isOmpEnabled",        globalEnvPtr->isMultithreaded},
         {"isCuQuantumEnabled",  globalEnvPtr->isCuQuantumEnabled},
@@ -397,13 +404,19 @@ extern "C" {
 
 void initCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMultithread) {
 
-    validateAndInitCustomQuESTEnv(useDistrib, useGpuAccel, useMultithread, __func__);
+    const bool userOwnsMpi = false;
+    validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__);
 }
 
 
+void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread) {
+    validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__);
+}
+
 void initQuESTEnv() {
 
-    validateAndInitCustomQuESTEnv(modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__);
+    const bool userOwnsMpi = false;
+    validateAndInitCustomQuESTEnv(modeflag::USE_AUTO, userOwnsMpi, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__);
 }
 
 
@@ -436,7 +449,7 @@ void finalizeQuESTEnv() {
 
     if (globalEnvPtr->isDistributed) {
         comm_sync();
-        comm_end();
+        comm_end(globalEnvPtr->userOwnsMpi);
     }
 
     // free global env's heap memory and flag it as unallocated
@@ -454,8 +467,12 @@ void syncQuESTEnv() {
     if (globalEnvPtr->isGpuAccelerated)
         gpu_sync();
 
-    if (globalEnvPtr->isDistributed)
+    if (globalEnvPtr->isDistributed) {
         comm_sync();
+        #if COMPILE_SUBCOMM
+            updateQuESTEnvDistInfo();
+        #endif
+    }
 }
 
 
@@ -498,10 +515,11 @@ void getEnvironmentString(char str[200]) {
     int cuQuantum = env.isGpuAccelerated && gpu_isCuQuantumCompiled();
     int gpuDirect = env.isGpuAccelerated && gpu_isDirectGpuCommPossible();
 
-    snprintf(str, 200, "CUDA=%d OpenMP=%d MPI=%d threads=%d ranks=%d cuQuantum=%d gpuDirect=%d",
+    snprintf(str, 200, "CUDA=%d OpenMP=%d MPI=%d userOwnsMPI=%d threads=%d ranks=%d cuQuantum=%d gpuDirect=%d",
         env.isGpuAccelerated,
         env.isMultithreaded,
         env.isDistributed,
+        env.userOwnsMpi,
         numThreads,
         env.numNodes,
         cuQuantum,

quest/src/api/subcommunicator.cpp

@@ -0,0 +1,31 @@
+#include "quest/include/config.h"
+#include "quest/include/environment.h"
+#include "quest/include/subcommunicator.h"
+
+#include "quest/src/comm/comm_config.hpp"
+#include "quest/src/core/errors.hpp"
+
+#if COMPILE_MPI && COMPILE_SUBCOMM
+
+#include <stdbool.h>
+#include <mpi.h>
+
+void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) {
+    // useDistrib and userOwnsMpi are implied by the user of this initialiser
+    const int useDistrib = 1;
+    const bool userOwnsMpi = true;
+
+    // set mpiCommQuest to user provided communicator
+    if (comm_isInit()) {
+        comm_setMpiComm(userQuestComm);
+    } else {
+        error_commNotInit();
+    }
+
+    // initialise QuEST around that communicator
+    initCustomMpiQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread);
+
+    return;
+}
+
+#endif