Refactor preconditioner implementation

.gitignore

@@ -5,3 +5,8 @@
 build
 builds
 .vscode
+
+# local build and tooling artifacts
+build.log
+Dockerfile
+.claude/

CMakeLists.txt

@@ -194,13 +194,14 @@ target_sources(
           src/MatrixWrapper/SparsityPattern.cpp
           src/MatrixWrapper/Distributed.cpp
           src/MatrixWrapper/Combination.cpp
+          src/Preconditioner/CoarseSolver.cpp
           src/Repartitioner.cpp
           src/Solver/CG.cpp
           src/Solver/PipeCG.cpp
           src/Solver/BiCGStab.cpp
           src/Solver/GMRES.cpp
           src/Solver/Multigrid.cpp
-  # src/Solver/IR.cpp
+          # src/Solver/IR.cpp
 )
 
 enable_sanitizers(
@@ -218,7 +219,13 @@ endif()
 if(${GINKGO_BUILD_CUDA})
   find_package(CUDAToolkit REQUIRED)
   target_compile_definitions(OGL PRIVATE GINKGO_BUILD_CUDA=1)
-  target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
+  if(TARGET CUDA::nvToolsExt)
+    target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
+  elseif(TARGET CUDA::nvtx3)
+    target_link_libraries(OGL PUBLIC CUDA::nvtx3)
+  else()
+    message(WARNING "Neither CUDA::nvToolsExt nor CUDA::nvtx3 found; NVTX annotations disabled")
+  endif()
 endif()
 
 if(OGL_DATA_VALIDATION)
@@ -243,15 +250,20 @@ add_custom_target(
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
   COMMENT "a target to format cmake files")
 
-install(
-  TARGETS OGL
-          ginkgo
-          ginkgo_device
-          ginkgo_hip
-          ginkgo_cuda
-          ginkgo_omp
-          ginkgo_dpcpp
-          ginkgo_reference
-  DESTINATION $ENV{FOAM_USER_LIBBIN})
+set(_ogl_install_targets OGL)
+foreach(
+  _target
+  ginkgo
+  ginkgo_device
+  ginkgo_hip
+  ginkgo_cuda
+  ginkgo_omp
+  ginkgo_dpcpp
+  ginkgo_reference)
+  if(TARGET ${_target})
+    list(APPEND _ogl_install_targets ${_target})
+  endif()
+endforeach()
+install(TARGETS ${_ogl_install_targets} DESTINATION $ENV{FOAM_USER_LIBBIN})
 
 include(cmake/banner.cmake)

cmake/AutoEnableDevice.cmake

@@ -33,5 +33,5 @@ if(NOT DEFINED OGL_BUILD_HIP)
         CACHE INTERNAL "")
   endif()
 else()
-  message(STATUS "Skip HIP detection OGL_HIP_CUDA=${OGL_BUILD_HIP}")
+  message(STATUS "Skip HIP detection OGL_BUILD_HIP=${OGL_BUILD_HIP}")
 endif()

cmake/CxxThirdParty.cmake

@@ -6,10 +6,13 @@
 
 include(cmake/CPM.cmake)
 
+
 if(NOT DEFINED OGL_GINKGO_DIR)
-  set(OGL_GINKGO_CHECKOUT_VERSION
-      "ogl_0600_gko190"
-      CACHE STRING "Use specific version of ginkgo")
+  if(NOT DEFINED OGL_GINKGO_CHECKOUT_VERSION)
+    set(OGL_GINKGO_CHECKOUT_VERSION
+        "ogl_0600_gko190"
+        CACHE STRING "Use specific version of ginkgo")
+  endif()
   message(STATUS "Using CPM to get Ginkgo ${GINKGO_CHECKOUT_VERSION}")
   set(OGL_GINKGO_VIA_CPM ON)
 else()

include/OGL/DevicePersistent/ExecutorHandler.hpp

@@ -71,6 +71,14 @@ struct DeviceIdHandler {
         return device_global_id % num_devices_per_node;
     }
 
+    /* @brief returns the owner rank on the global comm world communicator
+     */
+    label global_owner() const
+    {
+        label rank = Pstream::myProcNo();
+        return rank - (rank % ranks_per_gpu);
+    }
+
     /* @brief check if rank is an owning rank
      */
     bool is_owner() const
@@ -270,15 +278,16 @@ class ExecutorHandler
             // gko comm
             label group = device_id_handler_.compute_group();
             MPI_Comm gko_comm;
-            label host_rank = 0;
+            label host_rank = Pstream::myProcNo();
             MPI_Comm_split(MPI_COMM_WORLD, group, host_rank, &gko_comm);
             device_comm_ =
                 std::make_shared<gko::experimental::mpi::communicator>(
                     gko_comm, gko_force_host_buffer_);
 
             // repart comm
             MPI_Comm repart_comm;
-            label device_id = device_id_handler_.compute_device_id(4);
+            label global_rank = Pstream::myProcNo();
+            label device_id = global_rank / device_id_handler_.ranks_per_gpu;
             MPI_Comm_split(MPI_COMM_WORLD, device_id, host_rank, &repart_comm);
             repart_comm_ =
                 std::make_shared<gko::experimental::mpi::communicator>(
@@ -300,6 +309,15 @@ class ExecutorHandler
      * */
     bool get_non_orig_device_comm() const { return non_orig_device_comm_; }
 
+    label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
+
+    void set_ranks_per_gpu(label ranks_per_gpu)
+    {
+        device_id_handler_.ranks_per_gpu = ranks_per_gpu;
+    }
+
+    label get_owner_rank() const { return device_id_handler_.global_owner(); }
+
     const std::shared_ptr<gko::Executor> get_device_exec() const
     {
         return this->get_persistent_object();