From a1cea3ed049d944b6a1a13146c636cd7039271b9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 30 Jul 2025 09:27:02 +0200
Subject: [PATCH 01/12] use repartAllToAll

---
 include/OGL/CommunicationPattern.hpp |  6 ++++++
 src/CommunicationPattern.cpp         | 27 +++++++++++++++++++++++++++
 src/MatrixWrapper/Distributed.cpp    |  9 +++++----
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/include/OGL/CommunicationPattern.hpp b/include/OGL/CommunicationPattern.hpp
index 7fcf542e..0bb6e477 100644
--- a/include/OGL/CommunicationPattern.hpp
+++ b/include/OGL/CommunicationPattern.hpp
@@ -19,6 +19,12 @@ struct AllToAllPattern {
     std::vector<int> recv_offsets;
 };
 
+/* @brief computes AllToAllPattern for repart comm from global allToAll pattern
+ *
+ */
+AllToAllPattern compute_repart_allToall(
+    const ExecutorHandler &exec_handler, const AllToAllPattern allToAll);
+
 /* @brief  This function computes the send and recv counts vectors and the send
  * and recv offsets vectors for scattering from an owner to all ranks, including
  * owner itself
diff --git a/src/CommunicationPattern.cpp b/src/CommunicationPattern.cpp
index 192a7f6c..ba4c95e7 100644
--- a/src/CommunicationPattern.cpp
+++ b/src/CommunicationPattern.cpp
@@ -178,6 +178,33 @@ void communicate_values(const ExecutorHandler &exec_handler,
     // }
 }
 
+AllToAllPattern compute_repart_allToall(
+    const ExecutorHandler &exec_handler, const AllToAllPattern allToAllIn)
+    {
+    auto host_comm = exec_handler.get_host_comm();
+    auto repart_comm = exec_handler.get_repart_comm();
+    auto ranks = repart_comm->size();
+
+    std::vector<int> send_counts(ranks);
+    std::vector<int> send_offsets(ranks+1);
+    std::vector<int> recv_counts(ranks);
+    std::vector<int> recv_offsets(ranks+1);
+
+    label start_rank = host_comm->rank();
+    for (auto i=0;i<exec_handler.get_ranks_per_gpu();i++){
+        send_counts[i] = allToAllIn.send_counts[start_rank+i];
+        send_offsets[i] = allToAllIn.send_offsets[start_rank+i];
+        recv_counts[i] = allToAllIn.recv_counts[start_rank+i];
+        recv_offsets[i] = allToAllIn.recv_offsets[start_rank+i];
+    }
+
+    send_offsets.back() = allToAllIn.send_offsets.back();
+    recv_offsets.back() = allToAllIn.recv_offsets.back();
+    return {
+        send_counts, send_offsets, recv_counts, recv_offsets
+    };
+    }
+
 void communicate_values(
     std::shared_ptr<const gko::Executor> src_exec,
     std::shared_ptr<const gko::Executor> target_exec,
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index 72eaae43..312a7346 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -384,7 +384,7 @@ void update_impl(
     std::map<label, scalar *> linops, label verbose)
 {
     auto comm = exec_handler.get_host_comm();
-    // auto repart_comm = exec_handler.get_repart_comm();
+    auto repart_comm = exec_handler.get_repart_comm();
     auto ref_exec = exec_handler.get_ref_exec();
     auto rank = exec_handler.get_host_rank();
     auto device_exec = exec_handler.get_device_exec();
@@ -392,12 +392,13 @@ void update_impl(
     word fieldname = host_A->get_field_name();
 
     // perform all-to-all updates first
-    auto all_to_all_update = [comm, ref_exec, device_exec,
+    auto all_to_all_update = [repart_comm, ref_exec, device_exec,
                               all_to_all_update_data, host_A,
-                              force_host_buffer]() {
+                              force_host_buffer, exec_handler]() {
         for (auto [id, comm_pattern, data_ptr] : all_to_all_update_data) {
+            auto repartAllToAll = compute_repart_allToall(exec_handler, comm_pattern);
             auto [length, send_data_ptr] = host_A->get_interface_data(id);
-            communicate_values(ref_exec, device_exec, comm, comm_pattern,
+            communicate_values(ref_exec, device_exec, repart_comm, repartAllToAll,
                                send_data_ptr, data_ptr, force_host_buffer);
         }
     };

From b38c4415ff5c00a5f0da4461fc1267800033b1ce Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 4 Aug 2025 17:32:23 +0200
Subject: [PATCH 02/12] format

---
 src/CommunicationPattern.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/CommunicationPattern.cpp b/src/CommunicationPattern.cpp
index ba4c95e7..0c7fe548 100644
--- a/src/CommunicationPattern.cpp
+++ b/src/CommunicationPattern.cpp
@@ -171,7 +171,8 @@ void communicate_values(const ExecutorHandler &exec_handler,
     //     std::cout << __FILE__ << ":" << __LINE__ << " received "
     //               << std::to_string(recv_size) << " elements of "
     //               << std::to_string(recv_size * 8 / 1e9)
-    //               << "[Gb] on device rank: " << std::to_string(rank) << " time "
+    //               << "[Gb] on device rank: " << std::to_string(rank) << "
+    //               time "
     //               << std::to_string(delta_t) << "[ms] bandwidth: "
     //               << std::to_string(recv_size * 8 / delta_t / 1e6)
     //               << "[Gb/s] \n";

From 6e89aac3e398e580837c59cbbd10681021c93552 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 4 Aug 2025 17:35:52 +0200
Subject: [PATCH 03/12] init ginkgo comm

---
 src/Repartitioner.cpp         | 2 --
 test/unit/Repartitioner1D.cpp | 1 +
 test/unit/Repartitioner2D.cpp | 3 ++-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Repartitioner.cpp b/src/Repartitioner.cpp
index 33269aee..fe26ecc4 100644
--- a/src/Repartitioner.cpp
+++ b/src/Repartitioner.cpp
@@ -27,7 +27,6 @@ Repartitioner::repartition_sparsity(
 {
     LOG_1(verbose_, "start repartition sparsity pattern")
 
-
     auto exec = exec_handler.get_ref_exec();
     auto comm = *exec_handler.get_host_comm().get();
     label rank = exec_handler.get_host_rank();
@@ -67,7 +66,6 @@ Repartitioner::repartition_sparsity(
         return gather_closure(comm_pattern, tmp, offset);
     };
 
-
     /* Helper function, create and return gathered sparsity pattern based on
      * in_sparsity
      *  */
diff --git a/test/unit/Repartitioner1D.cpp b/test/unit/Repartitioner1D.cpp
index bec8af74..4305f025 100644
--- a/test/unit/Repartitioner1D.cpp
+++ b/test/unit/Repartitioner1D.cpp
@@ -136,6 +136,7 @@ TEST_P(RepartitionerFixture1D, can_repartition_sparsity_pattern)
     // Arrange
     auto ranks_per_gpu = GetParam();
     auto repartitioner = Repartitioner(local_size, ranks_per_gpu, 0, exec);
+    exec.init_device_comm();
 
     // std::vector<label> ranks{rank};
     auto local_sparsity = std::make_shared<SparsityPattern>();
diff --git a/test/unit/Repartitioner2D.cpp b/test/unit/Repartitioner2D.cpp
index accbfb67..030a01f6 100644
--- a/test/unit/Repartitioner2D.cpp
+++ b/test/unit/Repartitioner2D.cpp
@@ -161,6 +161,7 @@ INSTANTIATE_TEST_SUITE_P(RepartitionerFixture2DInstantiation,
 TEST_P(RepartitionerFixture2D, can_repartition_2D_comm_pattern_for_n_ranks)
 {
     // Arrange
+    exec.init_device_comm();
     auto ranks_per_gpu = GetParam();
     auto repartitioner = Repartitioner(local_size, ranks_per_gpu, 0, exec);
     auto ref_exec = exec.get_ref_exec();
@@ -209,7 +210,6 @@ TEST_P(RepartitionerFixture2D, can_repartition_2D_comm_pattern_for_n_ranks)
         exec, comm_target_ids[rank], rows[rank]);
 
     // Act
-    exec.init_device_comm();
     auto repart_comm_pattern =
         repartitioner.repartition_comm_pattern(exec, comm_pattern);
 
@@ -235,6 +235,7 @@ TEST_P(RepartitionerFixture2D, can_repartition_sparsity_pattern)
 {
     // Arrange
     auto ranks_per_gpu = GetParam();
+    exec.init_device_comm();
     auto repartitioner = Repartitioner(local_size, ranks_per_gpu, 0, exec);
     auto ref_exec = exec.get_ref_exec();
 

From 98f70689e9a5a83f5dce970e4a2c2571db0dd38b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 5 Aug 2025 07:26:22 +0200
Subject: [PATCH 04/12] fixup clang tidy .C -> .cpp

---
 .github/workflows/static_checks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml
index f5f51fdc..49e11674 100644
--- a/.github/workflows/static_checks.yaml
+++ b/.github/workflows/static_checks.yaml
@@ -67,7 +67,7 @@ jobs:
       run: |
         git config --global --add safe.directory /__w/OGL/OGL
         # Create list of all source files belonging to this repository
-        git ls-files | grep -E "\.(C)" > pattern
+        git ls-files | grep -E "\.(cpp)" > pattern
         # Create list of .cpp files that are in this repository and part of the
         # compilation database
         # also filters out " at the begin and end of the filename

From b262ac58008dfdf3a13935988b19c20d2bfbbb60 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 25 Sep 2025 15:15:38 +0200
Subject: [PATCH 05/12] remove broken aws workflow

---
 .github/workflows/compile_on_aws.yml | 152 ---------------------------
 1 file changed, 152 deletions(-)
 delete mode 100644 .github/workflows/compile_on_aws.yml

diff --git a/.github/workflows/compile_on_aws.yml b/.github/workflows/compile_on_aws.yml
deleted file mode 100644
index bdde16e7..00000000
--- a/.github/workflows/compile_on_aws.yml
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: 2024 OGL authors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-name: Compile on AWS
-run-name: Compile on AWS
-
-on:
-  pull_request:
-    types: synchronize
-jobs:
-  start-runner:
-    if: contains(github.event.pull_request.labels.*.name, 'full_ci')
-    name: Start self-hosted EC2 runner
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr
-          aws-region: us-east-1
-      - name: Start EC2 runner
-        id: start-ec2-runner
-        uses: HendriceH/ec2-github-runner@v1.10  # Starts 60GB Root + 30 GB Share volume
-        with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-03af087024bfdbbee # Deep learning AMI
-          ec2-instance-type: g4dn.xlarge
-          iam-role-name: Role4Github
-          subnet-id: subnet-b5d2adbb
-          security-group-id: sg-559f8967
-          aws-resource-tags: > # optional, requires additional permissions
-            [
-              {"Key": "ucfd-project", "Value": "BMBF_2022_EXASIM"},
-              {"Key": "ucfd-client", "Value": "UCFD-RD"},
-              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-            ]
-          pre-runner-script: |
-                 #!/bin/bash
-                 sudo yum update -y && \
-                 sudo yum install docker git libicu ninja-build libasan10 -y
-                 sudo amazon-linux-extras install epel -y
-                 sudo yum install Lmod -y
-                 sudo systemctl enable docker
-                 sudo curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
-                 sudo mkfs -t xfs /dev/sda1
-                 sudo mkdir -p /share
-                 sudo mount /dev/sda1 /share
-                 aws s3 cp s3://ucfd-share/pcluster/3.x/alinux2/x86_64/postinstall_github .
-                 chmod +x postinstall_github
-                 sudo ./postinstall_github > ~/install.log
-                 ln -s /share/software/cmake/3.27.8/share/cmake-3.27 /usr/share/cmake-3.27
-                 mkdir -p /share/ec2-user
-                 export USER=ec2-user
-  do-the-job:
-    name: Do the job on the runner
-    needs: start-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
-    steps:
-      - name: Checkout OGL
-        uses: actions/checkout@v2
-        with:
-          ref: dev
-      - name: Test env
-        shell: bash -o pipefail -i {0}
-        run: |
-          export HOME=/share/ec2-user
-          module load gnu/10
-          module load libfabric-aws
-          module load openmpi
-          module load OpenFOAM
-          module load cmake
-          env
-      - name: Install OBR
-        shell: bash -o pipefail -i {0}
-        run: |
-          module load conda
-          conda activate
-          python3 -m pip install --upgrade pip
-          pip install setuptools --upgrade
-          git clone https://github.com/exasim-project/OBR
-          cd OBR
-          pip install .
-          obr --version
-
-      - name: Config
-        if: always()
-        shell: bash -o pipefail -i {0}
-        run: |
-          export HOME=/share/ec2-user
-          module load gnu/10
-          module load libfabric-aws
-          module load openmpi
-          module load OpenFOAM
-          module load cmake
-          cmake --list-preset
-          cmake -DOGL_CUDA_ARCHITECTURES=52 --preset ninja-cuda-release
-      - name: Build and install
-        shell: bash -o pipefail -i {0}
-        run: |
-          export HOME=/share/ec2-user
-          module load gnu/10
-          module load libfabric-aws
-          module load openmpi
-          module load OpenFOAM
-          module load cmake
-          cmake --build --preset ninja-cuda-release  --target install
-      - name: Run integration tests
-        shell: bash -o pipefail -i {0}
-        run: |
-          export HOME=/share/ec2-user
-          module load conda
-          conda activate
-          module load gnu/10
-          module load libfabric-aws
-          module load openmpi
-          module load OpenFOAM
-          export GINKGO_EXECUTOR=cuda
-          obr init -g --config test/integration.yaml
-          obr run -o runParallelSolver
-          obr status
-
-  stop-runner:
-    name: Stop self-hosted EC2 runner
-    needs:
-      - start-runner # required to get output from the start-runner job
-      - do-the-job # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr
-          aws-region: us-east-1
-      - name: Stop EC2 runner
-        uses: HendriceH/ec2-github-runner@v1.10
-        with:
-          mode: stop
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          label: ${{ needs.start-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

From c6d65de0caad018177aad63183028f18ba33de27 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 30 Jul 2025 09:35:50 +0200
Subject: [PATCH 06/12] add get_ranks_per_gpu

---
 include/OGL/DevicePersistent/ExecutorHandler.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index 3c9b7da5..50ec322e 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -300,6 +300,8 @@ class ExecutorHandler
      * */
     bool get_non_orig_device_comm() const { return non_orig_device_comm_; }
 
+    label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
+
     const std::shared_ptr<gko::Executor> get_device_exec() const
     {
         return this->get_persistent_object();

From 809c61afe6b7a04cc2e0953628115f698ac1bb8e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <go@hpsim.de>
Date: Wed, 30 Jul 2025 13:57:01 +0200
Subject: [PATCH 07/12] add basic implementation of gatherv instead off
 alltoallv for repart update

---
 include/OGL/.Preconditioner.hpp.swn           | Bin 0 -> 16384 bytes
 include/OGL/.Preconditioner.hpp.swo           | Bin 0 -> 16384 bytes
 .../OGL/DevicePersistent/ExecutorHandler.hpp  |   7 ++--
 include/OGL/lduLduBase.hpp                    |   1 +
 src/CommunicationPattern.cpp                  |   6 +--
 src/MatrixWrapper/Distributed.cpp             |  36 +++++++++++++++++-
 6 files changed, 42 insertions(+), 8 deletions(-)
 create mode 100644 include/OGL/.Preconditioner.hpp.swn
 create mode 100644 include/OGL/.Preconditioner.hpp.swo

diff --git a/include/OGL/.Preconditioner.hpp.swn b/include/OGL/.Preconditioner.hpp.swn
new file mode 100644
index 0000000000000000000000000000000000000000..6793705a6f00ad2117ddacb5b1735306a1af1c01
GIT binary patch
literal 16384
zcmeHN-)|hn9iO(L1WG7sA9&$ma!BP)v3(Z@hiKziRqZ;swXsP;04Zph+nu}HcyD(*
zJ7=Fs)1pfJ2c@n0Pzk|PrAn>TAL1#9N9ao{KnNrxeh4ZdfdqYODeY%=X7_g2mj<Fr
zRjXO)<Gb7MkNM7LzVn@(^>jPu*EZ;hxp{)?HbVZ$&J7L^?t1+FwR;Hv-rb#J$CcpU
zUp<-3j!ztq=I6rLW1$}pSrAPz`GI!W_L*dDFZOvm?uYFl^1>0mZ=POnZwc<jksrt)
zj<}c`B*_#v_LgP9GH_iC93sc&k9L&losGlv>tB1|y1ZB;%YbFTGGH073|Iy%1C{~H
zfMwu+lL46=BtJyxUo_!&&F2?tpZ_%1Idk8u$v-vn4RikwHTge`{EWGOz%-Dz^NEq)
zYvku^^1mDT{YL(an*47@e!<8;tjWKWw`)GXYdV~_Z*JQU%YbFTGGH073|Iy%1C{~H
zfMvikU>UFs{9iJ_d_um7p1q?s!ubDxzW;yetAzXscm{YH_zCbNFa~yk`+&m$1r7m!
zy_JxUfwzDcfv12JI01Zeh>$-6zXYBD9sw4BTY=YZA>;+%5^xNd2U@^E;6~stUm@f@
z;5pz5@H612z-8e3fCP>JZ{AGEZ-J+P$AJalX5fRHU=O?tTmhZ{wtzc;R}K>Ld*Ii=
z7+3&q01g1p-bl#Lfro)xfRDaR$gcnfJPO<nym$j4KLSnyA0Hs(dEgn~X<z_s0e1nP
z;^o8Zz-z#7fF}VDSOz}Adx+lwd%$_%0pK{GUdX61dHX-`f`XPVHj*(-dFYCGH>IaY
zNklVQWF#~@%b|a?mBn(*M9R;=>||jT^k}mejJdCcQ4sZ8^b)e7cFo06)8{*Z$6Y?=
zUaK{6QPZSzyMY|Ig8L&+odR)t%#*Qbrg6B#?~BE7xz*a|Uis*6$BvWAIB}daJPrI2
z3zhm}iW4bJEICeh6oh{Bn#OCA3#M8wEObOeHg*HZ(k8wsCEexL%)V%}z9*D+z?xT)
zCzYcvP5d-n-K&O}bzBs|6yCDI^0F~;wGKUHNEk4|o7Xi6ldg6#{M&9CJjR<xN7aZx
zwi%^#J4n#DkQn!-6eC_$>|VS|*rm=QXu;A#PY0k=Pxq?Sk&X4u({87^!$mhvxjRFL
zX}{W{3Jq5(eaDfpn@XV~(=0v6gZ@BH&%ByJb%f~=+PDy1kP|m@(4a?1)>QRR%}*(5
zb0-zo^fS6CPrg>}N$~JoY-Z(WbhK}(nx0i=cg5FkEPO>XhN*Fy<4hKloDN5!3~0!A
zct}%ku*<|_jv}b$K66yCl;vx+_ot|j6JzoADA^P%yvN|S*{G;AW@yN|JZvs>>NR7E
zX;C!v)I6(7hA3Pd<+I1jPIM(K%1WsgmZp|Y8uWfjbCY}+NVK90O|vuCIa+Q;CCEPG
z#j~h#-e9TOsBs2AJr<_Cl%`oUHMM%M)d8FuZAGOZWkH2njL|5?bfL9slCkJG%l*8R
zPHMN-?|FIU$r7V40?fYE>Z)LD&2TQ&T_pYC6dzmtynI*fE)@Z1$x&LVS`^0zQ(4Hf
z`IxGD=(~v&i{^acYPP2AEm2d1XS5mhgKW)VVvl~qpA`1Gmu#u|Hr6q$rDBPe7PhO~
zWO3>GydFfG$&&g$WGGN$Z8WtJqq-|t;%4N!*{pNTtkY|Y<ijLX^Kwz{B|Kt7PHX+t
zBYf#{!3(8%5zVV<xsK`tSdJqp;xKjf+Kcs0|K7LaRn_;;XhX6&)Va*O0kRAkh-ot6
zi&75)?pBrE^6Ke#-KO6y**G-aOI=N(tNM*lt2au@baLsjC=YhE3wb)|>@rMgf()^y
zFIZWSZ8R&AwU%=qarYwk<Qd)%uo$Vd>dug4)C~jA(W9bWRGM<ll)P4)RXy!uDvK3=
z`y;CQnbsR<BqG>Bl9{L{WPbi{S)ip7kWD8(PVlre<dI~d<M@iLpgWQrrKFk?j&oKo
zIn|P)Gf5XDJW!YdsegaI>Y{CiYRS{jl%@b~s8vrDH(k8)+P6;+g|ZjMOqw2QJ#U(t
zc{DBi3uMu3_9$CCjaHOIp=4rP6V>9Z8x4xWT&Hl&TdaiVcQ58H>#E;jUfc~9{TNBh
zt995=a828cj%lXXw5$X<l;b=|J?F6~rKOI0AxqP|^qCoo&4?-tX~H~CS7SC*yOTT3
zk#pJ*yuY@oC&b*njgp|Zv%I>ZZk6t-1L_-D#eNX=80p$xnWU*Yt6p}XlvlDH&ldI!
z&_iaozr|9n6(?>N`$4qbkK6h~Nte)QB3OUOXdHQ*kanA%-CB8Qb{+etDCM(jKAZ);
zfD4D7-ddkMHrJVr#jIKb1e6Q?)ga`{ak3}WgYP*$mJXfo%->BBo7(FQVv!QH|Nk-e
z@6Q2h|BvlzcD;e$uL3U{*%c+Xmu0{*U>UFsSOzQumI2FvWxz6E8L$jk1}p>r6$UO-
iiq`|xcO2xhep@to-z&HriHP>ie{k?JT>tm+B>5MH00&|K

literal 0
HcmV?d00001

diff --git a/include/OGL/.Preconditioner.hpp.swo b/include/OGL/.Preconditioner.hpp.swo
new file mode 100644
index 0000000000000000000000000000000000000000..59f5bb35988ed3c6ebd5b633ff6bf04be8b770a0
GIT binary patch
literal 16384
zcmeHOTa4UR89wPPEHvd(5agkfyQv}%*`1kON`f=H5^UJ8G?yi9LZvn`9{bFUvmV>A
zJ-f3Ffxht42MVodLE?oZedz-Qo@jUgfp`Iml&F=siR%O6iH9I0AY8t48y|aSv)u?G
zA#CYedwl-$pZ|R4Kj--W?QJ*DEFCuv)%Ot`?;zx-uPydoeeSNOzjn_<gq}ap-2VYq
zaO<PD5;hMu57vXobpkKyJAOE^!l#A8hUa8X!;L)Jh`K?;58YsZ>yt;1HI@_VMxp0t
zeiYKA-izajimICx0~G^XV_=YVW)JN+y?@`{Cabt>`a$FVcb(pvj!L>>pkkn6pkkn6
zpkkn6pkkn6pkm;Ent?3dPCkPk+#!3hBG21O=YPrLjJ)43mH$(gFU$MiFO~m8mY<UM
z@0Sgzc5ce@M`SrImH%CqpOxibFO}aZ?|)F9cgPMY{qlBosu-vks2Hdis2Hdis2Hdi
zs2Hdis2Hdis2KRaWWezV`8ayMlf}z;|F7cz*WXUaHQ)+x8F&HsA}|EjfD^!jfC20P
z{(K)HH-O&(KLWlAq`)EI<_<!x11|%g1<nEw1NQ-M+)Kz0fK{Ll%mNPqcLH|+zkeHa
z0^bI%0?z}_0YhL7h=AR|FW*YYE5KKP3qTXN8~EcC>I1(8t^!X3j{xrle)ASWehyp$
zIsgS81`Yr_fxm1g<a@vXI0`HR4*=VMU))2;Gr&1u8n}5kA>RR>0X_-L0WIL2z>n`D
z<h#HN!1KU)AOx0x-N5TeUi<`j4frPTWuOmy2-pqWz}Q>|UIwlJmw`_KCjd5XzebYd
zSCbPRhCI^Dv#hk|B-FFxENS5@i!?L3E0JX#^TU(zJiG2YS>g|82&<FIipJJ!mYUKV
zSxcPQPTk&`lRRbeb~4(J11F<KD_f6g==7-}KLS~F+$>7g=iy^GF7m(ZN*%NFU5bxH
zP}dyUUaK7@s$8AxQD}N})pw~)ht#c&3#|EB&vqjxNvWMi!75G6!mk>cnyPc5V6UC_
zqbTb^q~ioB-9%eJS7~6z-M-y%Hf))HHlrGj#TPI|y@``rR(s$Fo|zXUG#ezL7(+g?
zCV`z(U;uwR?x=#WJYDg@e3WE<*fq0cKx^W@>}V3LrDjS4n_bj4q|6y)ks-Td%oW<6
zQnl+T%{bGLO$yAzvWqZoB-n)@`xeNoJ>j2)x<mEm`+W#qAXA+vF-+c?Ki53$w+eMq
z|0!zL4*Prdu#uSxh0&<UD=1NApOYlcdMj|+G?-_eVp*qXhbA<1=||uX%u3606UrQ^
zHY1e?okE3}7R=o;TQvPd^K9eLVd0p{&Dgoj+XQb^;$xyjm6Xq7oG@_*F8eHsc@iUb
zFP+_=JA+<{_c6_)nDsREy@3;)h?0IQ#n)L*GCy*R4O`S_i&zbG_3T<}Vku*D5QdHP
z3WNL3(DqT5ntQoFjkPBIFJVHyBUXcQJG$cw`SN6ubgYm0>^L_<ft+CS6SHEoV)BnV
zGBTO_Rbfd`&ZQ<^*>LUm*hr5rQMxo=XBc^Hbah2IT$+Jl*LIyWYjNraewgjwHxFvZ
zyE&$8eCiWTMxVs(a~t#>q$#3}x!msuDOM9hur^|ZFQAsS&{YnHi83W5JFb%I4}#3^
zCO&qhMSn=W1-|-*P$z567-GieWC;~md?x5v?ke{#*1It3X?YtfZ8C;hcMZNXZdi@8
zua=E@Lz^l2L@775`Chx(j>*}w`DSu^yXOUc`^5g%j<D64@<MkZF_sHam?cq=+Gg=L
zmPg%l@RYy@TZ!WX&}d~4pG=O>j)MU<r;C$q!Et*S;prIzeygi;Q9$jYu&qx`uWg%N
zTdQLk>3V95mDy%b4KTHMLlf3)7GvT0d?|I3bz^t23o~N;h4UE&f~3c&g+pjsixRJF
ziqa9a&rvr!MZ12Qv6j5{42cKrz;`V%g@g+3=c*}N?kbzvS(FnhCbC%2H*Qf;&!pZ!
zB2N5O?2giEjm<4zmgh8|PIMS!2k6r<a{|lq7+ZdOkWs7+MVGLwN5!LKag7k3B$fwk
z7+9c~UE;UX?+)#)tr?<XzpIuR;UJcETrF8*NfKfmcT5foUnhv1Y+jd}C?+>#UCz%&
z&GHR=w7V9Bl97;HY?qRJvCTX0kCxjqk4tW=V4l}Lr&N|jKjyem+i!UhJeL+%S4qL9
zbvYfAs%~1dU}QfVDkH_j$0n0%9d`pKO;z+Dr>Sv)u+<@r9hVx5k<-TwvP*t)ZcHb%
zyL3{_iuy;VwSvy-!r~EjOS{KDP(RK?q)#A>fzLLM<i5(jRo4|T;UhfiSjICD<lx?o
z4?8Ipg1OpO*AG{^QA3<qDI%U-apH9QjuGJ@m5@fmcy#&5<Fm&Q!-OfFUGkvJ@Ax#a
zjHAoPX7|^dvr#h3!l(qoBl$%?pbJsFp7`Bf_81*zma(t7?|=c@3_Q~Gq9i3O{$EFo
zx{er@#sBL4`74O&&jSa6ZNPQJ@z;QF03Cn=EVlmxV))CzCE!V*2K)?h`?J7fz(L?|
zh|j+TTmn81WPlAk09;3m{u0mzb^`AM-au^rCE!KiNnj7K9r!C^@;?DT1fB&>0Pg|b
z4ctIX{t|EncpTUP+zVVozpetW0xtqz06qt>{tkias2Hdis2Hdi_%AbX(J-*D6*p&c
zQ9S14(TYlP$xI)Tmm)tVA8XmOP8^BU4APoN3K4dT9Z)_<slk#qti-GtNL@Lrk?$FU
z5G9_m>LBlchZ+OfNEUhKajnqsrvIuu<j6YB44yiOBRsf-bj>I~5wHgW5z&h*tf`(x
z%-G3#GX~-w#4$v-#1DP+sPKPswvQaD$*AO^2M^t5SX?h2lb}XRf94Ev)5dJq<9i-a
zuxBoi7~=T>83-Q1m18YiSemP5DZtGuz*?ctW&uFtX|%B(BL?DKDTkFRmp3hIBc5z(
zRrRQ<s9#0~On2^K<G<J)>35Q6=0SPqSUEeiLe7y$Mvt6eyn^rrF?38NYC*@U8qK=$
zRdj~%7D|7%Ji@B}Mk8%(=uG1V{6~gV8FZUNt5wI_i#+PZ<drU@4Y3kt=gsEbyr~8g
zc~j)N7}J_dDOG9!M?;1Y#1yTMUSh}uhz$HYEuw0&^F@Q=jR67>{U*DX&W6*t&lDY1
z)tr1+mCnu&V&v7S%8#r%!GOOcVX?JX6DD*{)#fp|lZs{OG*={|o-<6%ya_Y)jFGB#
zi~i*z`IT6+MK)KMWKMc{lUK;EMA$6jNflP|)V|g~lFLuNDs=M}DG1T`;z*V?P9}Uq
z6ADdFU|7#_*T!d}-nvqSyoVSWomY{$xFxULqxJUS(L8_2S5Z5TkZ7ifmU))@)Wp8|
z9m8aA@r=1SV_K$arfV$X)=k#0H(CeGX{HAYJIhF!xvmY|mglWR%ahw;ma#Wm%(7<f
smJ?1lZ**OH`7=IEN0R!5J{HQyi7udD-@aFibx?FqWb@VReG_i|Z>mC^761SM

literal 0
HcmV?d00001

diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index 50ec322e..43fd80ea 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -270,7 +270,7 @@ class ExecutorHandler
             // gko comm
             label group = device_id_handler_.compute_group();
             MPI_Comm gko_comm;
-            label host_rank = 0;
+            label host_rank = Pstream::myProcNo();
             MPI_Comm_split(MPI_COMM_WORLD, group, host_rank, &gko_comm);
             device_comm_ =
                 std::make_shared<gko::experimental::mpi::communicator>(
@@ -278,8 +278,9 @@ class ExecutorHandler
 
             // repart comm
             MPI_Comm repart_comm;
-            label device_id = device_id_handler_.compute_device_id(4);
-            MPI_Comm_split(MPI_COMM_WORLD, device_id, host_rank, &repart_comm);
+            label global_rank = Pstream::myProcNo();
+            label device_id = global_rank / device_id_handler_.ranks_per_gpu;
+            MPI_Comm_split(MPI_COMM_WORLD, device_id, host_rank , &repart_comm);
             repart_comm_ =
                 std::make_shared<gko::experimental::mpi::communicator>(
                     repart_comm, gko_force_host_buffer_);
diff --git a/include/OGL/lduLduBase.hpp b/include/OGL/lduLduBase.hpp
index e584adbe..33c86b07 100644
--- a/include/OGL/lduLduBase.hpp
+++ b/include/OGL/lduLduBase.hpp
@@ -216,6 +216,7 @@ class lduLduBase : public OGL_Info,
                                            solverPerformance &solverPerf) const
     {
         bool fused = solver_controls_.lookupOrDefault<Switch>("fuse", true);
+        exec_handler_.init_device_comm();
 
         auto repartitioner = std::make_shared<Repartitioner>(
             host_matrix_wrapper_->get_local_nrows(), ranks_per_gpu_, verbose_,
diff --git a/src/CommunicationPattern.cpp b/src/CommunicationPattern.cpp
index 0c7fe548..7a3387ba 100644
--- a/src/CommunicationPattern.cpp
+++ b/src/CommunicationPattern.cpp
@@ -192,15 +192,15 @@ AllToAllPattern compute_repart_allToall(
     std::vector<int> recv_offsets(ranks+1);
 
     label start_rank = host_comm->rank();
-    for (auto i=0;i<exec_handler.get_ranks_per_gpu();i++){
+    for (auto i=0;i<ranks;i++){
         send_counts[i] = allToAllIn.send_counts[start_rank+i];
         send_offsets[i] = allToAllIn.send_offsets[start_rank+i];
         recv_counts[i] = allToAllIn.recv_counts[start_rank+i];
         recv_offsets[i] = allToAllIn.recv_offsets[start_rank+i];
     }
 
-    send_offsets.back() = allToAllIn.send_offsets.back();
-    recv_offsets.back() = allToAllIn.recv_offsets.back();
+    send_offsets.back() = scalar(allToAllIn.send_offsets.back());
+    recv_offsets.back() = scalar(allToAllIn.recv_offsets.back());
     return {
         send_counts, send_offsets, recv_counts, recv_offsets
     };
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index 312a7346..31cacb53 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -396,10 +396,42 @@ void update_impl(
                               all_to_all_update_data, host_A,
                               force_host_buffer, exec_handler]() {
         for (auto [id, comm_pattern, data_ptr] : all_to_all_update_data) {
+
+	    // auto start = std::chrono::steady_clock::now();
             auto repartAllToAll = compute_repart_allToall(exec_handler, comm_pattern);
+	    // auto end = std::chrono::steady_clock::now();
+	    // auto delta_t = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count()/1000.0;
+	    // std::cout << __FILE__ << ":" << "delta t " << delta_t << " [ms]\n";
+
             auto [length, send_data_ptr] = host_A->get_interface_data(id);
-            communicate_values(ref_exec, device_exec, repart_comm, repartAllToAll,
-                               send_data_ptr, data_ptr, force_host_buffer);
+            // communicate_values(ref_exec, device_exec, repart_comm, repartAllToAll,
+            //                    send_data_ptr, data_ptr, force_host_buffer);
+	    // if ( repart_comm->rank() == 0 ) {
+		    // std::cout << __FILE__ <<
+			//     " Pstream::rank " << Pstream::myProcNo() <<
+			//     " repart_rank() "  << repart_comm->rank() <<
+			//     " send_offsets.back() "  <<  repartAllToAll.send_offsets.back() <<
+			//     " recv_counts: "  <<  repartAllToAll.recv_counts <<
+			//     " recv_offsets: "  <<  repartAllToAll.recv_offsets <<
+		    // std::endl;
+	    // }
+
+	    MPI_Request request;
+
+	    MPI_Igatherv(
+		    send_data_ptr,
+		    repartAllToAll.send_offsets.back(),
+		    MPI_DOUBLE,
+		    data_ptr,
+		    repartAllToAll.recv_counts.data(),
+		    repartAllToAll.recv_offsets.data(),
+		    MPI_DOUBLE,
+		    0,
+		    repart_comm->get(),
+                    &request
+		    );
+
+	    MPI_Wait(&request, MPI_STATUS_IGNORE);
         }
     };
 

From b9cfa1de9ad5f58531a7d248162fda6a7abb847c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <go@hpsim.de>
Date: Wed, 30 Jul 2025 17:03:34 +0200
Subject: [PATCH 08/12] use forceHostBuffer for vector updates

---
 include/OGL/DevicePersistent/Vector.hpp | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp
index 4d75f21b..6e28f3fc 100644
--- a/include/OGL/DevicePersistent/Vector.hpp
+++ b/include/OGL/DevicePersistent/Vector.hpp
@@ -67,7 +67,7 @@ struct VectorInitFunctor {
         //// TODO store
         auto comm_pattern = compute_gather_to_owner_counts(
             exec_, repartitioner->get_ranks_per_gpu(), host_size);
-        bool host_buffer = !exec_.get_non_orig_device_comm();
+        bool host_buffer = exec_.get_gko_force_host_buffer();
 
         communicate_values(ref_exec, exec, comm, comm_pattern,
                            host_view.get_const_data(),
@@ -171,13 +171,14 @@ class PersistentVector
 
     /** Copies the content of the distributed vector back to the original source
      **/
-    void copy_back()
+    MPI_Request copy_back()
     {
         auto exec = exec_.get_device_exec();
         auto rank = exec_.get_host_rank();
         auto ref_exec = exec_.get_ref_exec();
         auto comm = exec_.get_host_comm();
-        bool host_buffer = !exec_.get_non_orig_device_comm();
+	auto repart_comm = exec_.get_repart_comm();
+        bool host_buffer = exec_.get_gko_force_host_buffer();
 
         auto repartitioner = dist_matrix_->get_repartitioner();
         auto host_size = repartitioner->get_orig_size();
@@ -185,10 +186,25 @@ class PersistentVector
 
         auto comm_pattern = compute_scatter_from_owner_counts(
             exec_, repartitioner->get_ranks_per_gpu(), host_size);
+        auto repartAllToAll = compute_repart_allToall(exec_, comm_pattern);
 
         communicate_values(exec, ref_exec, comm, comm_pattern,
                            get_vector()->get_local_values(),
                            const_cast<T *>(memory_), host_buffer);
+        MPI_Request request;
+	// MPI_Iscatterv(
+        //             get_vector()->get_local_values(),
+	// 	    repartAllToAll.send_counts.data(),
+	// 	    repartAllToAll.send_offsets.data(),
+	// 	    MPI_DOUBLE,
+	// 	    const_cast<T *>(memory_),
+	// 	    repartAllToAll.recv_counts.back(),
+	// 	    MPI_DOUBLE,
+	// 	    0,
+	// 	    repart_comm->get(),
+        //             &request
+	// 	    );
+	return request;
     }
 
     /** Writes the content of the distributed vector to disk

From 56754def0a1624bd1faf13cc3a2d06b83ba5bc50 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <go@hpsim.de>
Date: Mon, 4 Aug 2025 08:28:48 +0200
Subject: [PATCH 09/12] performance tuning

---
 include/OGL/CommunicationPattern.hpp          |  5 +-
 .../OGL/DevicePersistent/ExecutorHandler.hpp  | 12 +++-
 include/OGL/DevicePersistent/Vector.hpp       | 69 +++++++++++++------
 include/OGL/StoppingCriterion.hpp             | 27 ++++++--
 include/OGL/lduLduBase.hpp                    | 13 +++-
 src/CommunicationPattern.cpp                  | 35 +++++-----
 src/MatrixWrapper/Distributed.cpp             | 65 ++++++++---------
 src/StoppingCriterion.cpp                     |  9 +++
 8 files changed, 152 insertions(+), 83 deletions(-)

diff --git a/include/OGL/CommunicationPattern.hpp b/include/OGL/CommunicationPattern.hpp
index 0bb6e477..64f1e221 100644
--- a/include/OGL/CommunicationPattern.hpp
+++ b/include/OGL/CommunicationPattern.hpp
@@ -22,8 +22,9 @@ struct AllToAllPattern {
 /* @brief computes AllToAllPattern for repart comm from global allToAll pattern
  *
  */
-AllToAllPattern compute_repart_allToall(
-    const ExecutorHandler &exec_handler, const AllToAllPattern allToAll);
+AllToAllPattern compute_repart_allToall(const ExecutorHandler &exec_handler,
+                                        const AllToAllPattern allToAll,
+                                        label start_rank);
 
 /* @brief  This function computes the send and recv counts vectors and the send
  * and recv offsets vectors for scattering from an owner to all ranks, including
diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index 43fd80ea..ec91a5db 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -71,6 +71,14 @@ struct DeviceIdHandler {
         return device_global_id % num_devices_per_node;
     }
 
+    /* @brief returns the owner rank on the global comm world communicator
+     */
+    label global_owner() const
+    {
+        label rank = Pstream::myProcNo();
+        return rank - (rank % ranks_per_gpu);
+    }
+
     /* @brief check if rank is an owning rank
      */
     bool is_owner() const
@@ -280,7 +288,7 @@ class ExecutorHandler
             MPI_Comm repart_comm;
             label global_rank = Pstream::myProcNo();
             label device_id = global_rank / device_id_handler_.ranks_per_gpu;
-            MPI_Comm_split(MPI_COMM_WORLD, device_id, host_rank , &repart_comm);
+            MPI_Comm_split(MPI_COMM_WORLD, device_id, host_rank, &repart_comm);
             repart_comm_ =
                 std::make_shared<gko::experimental::mpi::communicator>(
                     repart_comm, gko_force_host_buffer_);
@@ -303,6 +311,8 @@ class ExecutorHandler
 
     label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
 
+    label get_owner_rank() const { return device_id_handler_.global_owner(); }
+
     const std::shared_ptr<gko::Executor> get_device_exec() const
     {
         return this->get_persistent_object();
diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp
index 6e28f3fc..1b76273b 100644
--- a/include/OGL/DevicePersistent/Vector.hpp
+++ b/include/OGL/DevicePersistent/Vector.hpp
@@ -177,7 +177,7 @@ class PersistentVector
         auto rank = exec_.get_host_rank();
         auto ref_exec = exec_.get_ref_exec();
         auto comm = exec_.get_host_comm();
-	auto repart_comm = exec_.get_repart_comm();
+        auto repart_comm = exec_.get_repart_comm();
         bool host_buffer = exec_.get_gko_force_host_buffer();
 
         auto repartitioner = dist_matrix_->get_repartitioner();
@@ -186,25 +186,54 @@ class PersistentVector
 
         auto comm_pattern = compute_scatter_from_owner_counts(
             exec_, repartitioner->get_ranks_per_gpu(), host_size);
-        auto repartAllToAll = compute_repart_allToall(exec_, comm_pattern);
-
-        communicate_values(exec, ref_exec, comm, comm_pattern,
-                           get_vector()->get_local_values(),
-                           const_cast<T *>(memory_), host_buffer);
-        MPI_Request request;
-	// MPI_Iscatterv(
-        //             get_vector()->get_local_values(),
-	// 	    repartAllToAll.send_counts.data(),
-	// 	    repartAllToAll.send_offsets.data(),
-	// 	    MPI_DOUBLE,
-	// 	    const_cast<T *>(memory_),
-	// 	    repartAllToAll.recv_counts.back(),
-	// 	    MPI_DOUBLE,
-	// 	    0,
-	// 	    repart_comm->get(),
-        //             &request
-	// 	    );
-	return request;
+
+        label owner_rank = exec_.get_owner_rank();
+        auto repartAllToAll =
+            compute_repart_allToall(exec_, comm_pattern, owner_rank);
+
+        // if (owner_rank != Pstream::myProcNo()){
+        // 	label recv_count = repartAllToAll.recv_counts[0];
+        // 	repartAllToAll.recv_counts[Pstream::myProcNo()] = recv_count;
+        // 	repartAllToAll.recv_counts[0] = 0;
+        // }
+
+        // communicate_values(exec, ref_exec, comm, comm_pattern,
+        //                    get_vector()->get_local_values(),
+        //                    const_cast<T *>(memory_), host_buffer);
+        // std::cout << __FILE__
+        // 	<< " owner_rank " << exec_.get_owner_rank()
+        // 	<< " rank " << Pstream::myProcNo()
+        // 	<< " comm_pattern.send_counts: " << comm_pattern.send_counts
+        // 	<< " comm_pattern.recv_counts: " << comm_pattern.recv_counts
+        // 	<< " send_counts: " << repartAllToAll.send_counts
+        // 	<< " send_offsets: " << repartAllToAll.send_offsets
+        // 	<< " recv_counts: " << repartAllToAll.recv_counts
+        //        	<< "\n";
+
+        // auto start_rep = std::chrono::steady_clock::now();
+        label send_size = comm_pattern.send_offsets.back();
+        auto send_view = gko::array<scalar>::const_view(
+            exec, send_size, get_vector()->get_local_values());
+        auto tmp = gko::array<scalar>(exec, send_size);
+
+        tmp = send_view;
+        tmp.set_executor(ref_exec);
+        // auto end_rep = std::chrono::steady_clock::now();
+        // auto delta_t_rep =
+        // std::chrono::duration_cast<std::chrono::microseconds>(end_rep -
+        // start_rep).count() /1000.0; std::cout << __FILE__ << " copy back: "
+        // << delta_t_rep << " [ms]\n";
+
+
+        MPI_Request copy_back_req;
+        MPI_Iscatterv(tmp.get_data(),
+                      // get_vector()->get_local_values(),
+                      repartAllToAll.send_counts.data(),
+                      repartAllToAll.send_offsets.data(), MPI_DOUBLE,
+                      const_cast<T *>(memory_), repartAllToAll.recv_counts[0],
+                      MPI_DOUBLE, 0, repart_comm->get(), &copy_back_req);
+        MPI_Wait(&copy_back_req, MPI_STATUS_IGNORE);
+        return copy_back_req;
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/include/OGL/StoppingCriterion.hpp b/include/OGL/StoppingCriterion.hpp
index 0f70bd86..b32cd791 100644
--- a/include/OGL/StoppingCriterion.hpp
+++ b/include/OGL/StoppingCriterion.hpp
@@ -147,6 +147,8 @@ class StoppingCriterion {
 
     const label frequency_;
 
+    const word frequencyMode_;
+
     const scalar relaxationFactor_;
 
     const bool adapt_minIter_;
@@ -172,6 +174,8 @@ class StoppingCriterion {
           norm_eval_limit_(
               controlDict.lookupOrDefault("normEvalLimit", label(100))),
           frequency_(controlDict.lookupOrDefault("evalFrequency", label(1))),
+          frequencyMode_(controlDict.lookupOrDefault(
+              "evalFrequencyMode", word("relative"))),  // optimizer, fixed
           relaxationFactor_(
               controlDict.lookupOrDefault("relaxationFactor", scalar(0.6))),
           adapt_minIter_(
@@ -199,19 +203,32 @@ class StoppingCriterion {
     {
         label minIter = minIter_;
         label frequency = frequency_;
+        // in case of export_res all residuals need to be computed
+        std::cout << __FILE__ << "adapt minIter and frequency0 \n";
         if (!export_res) {
+            std::cout << __FILE__ << "adapt minIter and frequency1 \n";
             if (prev_solve_iters > 0 && adapt_minIter_ && prev_rel_cost > 0) {
+                std::cout << __FILE__ << "adapt minIter and frequency2 \n";
                 minIter = prev_solve_iters * relaxationFactor_;
-                auto alpha =
-                    sqrt(1.0 / (prev_solve_iters * (1.0 - relaxationFactor_)) *
-                         prev_rel_cost);
-                frequency = min(norm_eval_limit_, max(1, label(1 / alpha)));
+                if (frequencyMode_ == "optimizer") {
+                    auto alpha = sqrt(
+                        1.0 / (prev_solve_iters * (1.0 - relaxationFactor_)) *
+                        prev_rel_cost);
+                    frequency = min(norm_eval_limit_, max(1, label(1 / alpha)));
+                }
+                // if (frequencyMode_ == "relative") {
+                frequency = label(prev_solve_iters * 0.075) + 1;
+                // }
             }
         }
 
         word msg = "Creating stopping criterion with minIter " +
                    std::to_string(minIter) + " frequency " +
-                   std::to_string(frequency);
+                   std::to_string(frequency) + " prev_solve_iters " +
+                   std::to_string(prev_solve_iters) + +" adapt_minIter_  " +
+                   std::to_string(adapt_minIter_) + +" prev_rel_cost  " +
+                   std::to_string(prev_rel_cost) + " prev_solve_iters*0.075  " +
+                   std::to_string(prev_solve_iters * 0.075);
 
         MLOG_0(verbose, msg)
 
diff --git a/include/OGL/lduLduBase.hpp b/include/OGL/lduLduBase.hpp
index 33c86b07..9bc5c0a8 100644
--- a/include/OGL/lduLduBase.hpp
+++ b/include/OGL/lduLduBase.hpp
@@ -306,8 +306,15 @@ class lduLduBase : public OGL_Info,
             delta_t_solve_ = delta_t_solve;
         }
 
+        // auto start_rep = std::chrono::steady_clock::now();
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
+        // auto copy_back_req = dist_x.copy_back();
+        // auto end_rep = std::chrono::steady_clock::now();
+        // auto delta_t_rep =
+        // std::chrono::duration_cast<std::chrono::microseconds>(end_rep -
+        // start_rep).count() /1000.0; std::cout << __FILE__ << " full copy
+        // back: " << delta_t_rep << " [ms]\n";
 
         auto bandwidth_copy_back =
             sizeof(scalar) * psi.size() / delta_t_copy_x_back / 1000.0;
@@ -335,10 +342,12 @@ class lduLduBase : public OGL_Info,
             std::to_string(time_per_dof) + std::string(" [ns]") +
             std::string("\n\tTime per iteration and DOF: ") +
             std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
-            std::string("\n\tRetrieve results bandwidth ") +
-            std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
+            std::string("\n\tRetrieve results bandwidth ");  // +
+        std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
         MLOG_0(verbose_, msg)
 
+        // MPI_Wait(&copy_back_req,MPI_STATUS_IGNORE);
+
         return solverPerf;
     }
 
diff --git a/src/CommunicationPattern.cpp b/src/CommunicationPattern.cpp
index 7a3387ba..46502464 100644
--- a/src/CommunicationPattern.cpp
+++ b/src/CommunicationPattern.cpp
@@ -179,32 +179,31 @@ void communicate_values(const ExecutorHandler &exec_handler,
     // }
 }
 
-AllToAllPattern compute_repart_allToall(
-    const ExecutorHandler &exec_handler, const AllToAllPattern allToAllIn)
-    {
+AllToAllPattern compute_repart_allToall(const ExecutorHandler &exec_handler,
+                                        const AllToAllPattern allToAllIn,
+                                        label start_rank)
+{
     auto host_comm = exec_handler.get_host_comm();
     auto repart_comm = exec_handler.get_repart_comm();
     auto ranks = repart_comm->size();
 
-    std::vector<int> send_counts(ranks);
-    std::vector<int> send_offsets(ranks+1);
-    std::vector<int> recv_counts(ranks);
-    std::vector<int> recv_offsets(ranks+1);
-
-    label start_rank = host_comm->rank();
-    for (auto i=0;i<ranks;i++){
-        send_counts[i] = allToAllIn.send_counts[start_rank+i];
-        send_offsets[i] = allToAllIn.send_offsets[start_rank+i];
-        recv_counts[i] = allToAllIn.recv_counts[start_rank+i];
-        recv_offsets[i] = allToAllIn.recv_offsets[start_rank+i];
+    std::vector<int> send_counts(ranks, 0);
+    std::vector<int> send_offsets(ranks + 1, 0);
+    std::vector<int> recv_counts(ranks, 0);
+    std::vector<int> recv_offsets(ranks + 1, 0);
+
+    // label start_rank = host_comm->rank();
+    for (auto i = 0; i < ranks; i++) {
+        send_counts[i] = allToAllIn.send_counts[start_rank + i];
+        send_offsets[i] = allToAllIn.send_offsets[start_rank + i];
+        recv_counts[i] = allToAllIn.recv_counts[start_rank + i];
+        recv_offsets[i] = allToAllIn.recv_offsets[start_rank + i];
     }
 
     send_offsets.back() = scalar(allToAllIn.send_offsets.back());
     recv_offsets.back() = scalar(allToAllIn.recv_offsets.back());
-    return {
-        send_counts, send_offsets, recv_counts, recv_offsets
-    };
-    }
+    return {send_counts, send_offsets, recv_counts, recv_offsets};
+}
 
 void communicate_values(
     std::shared_ptr<const gko::Executor> src_exec,
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index 31cacb53..94448ed3 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -393,45 +393,40 @@ void update_impl(
 
     // perform all-to-all updates first
     auto all_to_all_update = [repart_comm, ref_exec, device_exec,
-                              all_to_all_update_data, host_A,
-                              force_host_buffer, exec_handler]() {
+                              all_to_all_update_data, host_A, force_host_buffer,
+                              exec_handler, rank]() {
         for (auto [id, comm_pattern, data_ptr] : all_to_all_update_data) {
-
-	    // auto start = std::chrono::steady_clock::now();
-            auto repartAllToAll = compute_repart_allToall(exec_handler, comm_pattern);
-	    // auto end = std::chrono::steady_clock::now();
-	    // auto delta_t = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count()/1000.0;
-	    // std::cout << __FILE__ << ":" << "delta t " << delta_t << " [ms]\n";
+            // auto start = std::chrono::steady_clock::now();
+            auto repartAllToAll =
+                compute_repart_allToall(exec_handler, comm_pattern, rank);
+            // auto end = std::chrono::steady_clock::now();
+            // auto delta_t =
+            // std::chrono::duration_cast<std::chrono::microseconds>(end -
+            // start).count()/1000.0; std::cout << __FILE__ << ":" << "delta t "
+            // << delta_t << " [ms]\n";
 
             auto [length, send_data_ptr] = host_A->get_interface_data(id);
-            // communicate_values(ref_exec, device_exec, repart_comm, repartAllToAll,
+            // communicate_values(ref_exec, device_exec, repart_comm,
+            // repartAllToAll,
             //                    send_data_ptr, data_ptr, force_host_buffer);
-	    // if ( repart_comm->rank() == 0 ) {
-		    // std::cout << __FILE__ <<
-			//     " Pstream::rank " << Pstream::myProcNo() <<
-			//     " repart_rank() "  << repart_comm->rank() <<
-			//     " send_offsets.back() "  <<  repartAllToAll.send_offsets.back() <<
-			//     " recv_counts: "  <<  repartAllToAll.recv_counts <<
-			//     " recv_offsets: "  <<  repartAllToAll.recv_offsets <<
-		    // std::endl;
-	    // }
-
-	    MPI_Request request;
-
-	    MPI_Igatherv(
-		    send_data_ptr,
-		    repartAllToAll.send_offsets.back(),
-		    MPI_DOUBLE,
-		    data_ptr,
-		    repartAllToAll.recv_counts.data(),
-		    repartAllToAll.recv_offsets.data(),
-		    MPI_DOUBLE,
-		    0,
-		    repart_comm->get(),
-                    &request
-		    );
-
-	    MPI_Wait(&request, MPI_STATUS_IGNORE);
+            // if ( repart_comm->rank() == 0 ) {
+            // std::cout << __FILE__ <<
+            //     " Pstream::rank " << Pstream::myProcNo() <<
+            //     " repart_rank() "  << repart_comm->rank() <<
+            //     " send_offsets.back() "  <<
+            //     repartAllToAll.send_offsets.back() << " recv_counts: "  <<
+            //     repartAllToAll.recv_counts << " recv_offsets: "  <<
+            //     repartAllToAll.recv_offsets <<
+            // std::endl;
+            // }
+            MPI_Request request;
+
+            MPI_Igatherv(send_data_ptr, repartAllToAll.send_offsets.back(),
+                         MPI_DOUBLE, data_ptr,
+                         repartAllToAll.recv_counts.data(),
+                         repartAllToAll.recv_offsets.data(), MPI_DOUBLE, 0,
+                         repart_comm->get(), &request);
+            MPI_Wait(&request, MPI_STATUS_IGNORE);
         }
     };
 
diff --git a/src/StoppingCriterion.cpp b/src/StoppingCriterion.cpp
index 38f9c13d..76865bd2 100644
--- a/src/StoppingCriterion.cpp
+++ b/src/StoppingCriterion.cpp
@@ -45,8 +45,16 @@ StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
         dist_vec::create(device_exec, comm, global_size, local_size));
     Axref->fill(0.0);
 
+    auto start_axref = std::chrono::steady_clock::now();
     compute_Axref_dist(global_size[0], local_size[0], device_exec, gkomatrix, x,
                        Axref);
+    auto end_axref = std::chrono::steady_clock::now();
+    auto delta_t_axref = std::chrono::duration_cast<std::chrono::microseconds>(
+                             end_axref - start_axref)
+                             .count() /
+                         1.0;
+    // std::cout << __FILE__ << " delta_t_axref " << delta_t_axref << " [mu
+    // s]\n";
 
     auto unity =
         gko::initialize<gko::matrix::Dense<scalar>>(1, {1.0}, device_exec);
@@ -169,6 +177,7 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
                               end_eval - start_eval)
                               .count() /
                           1.0;
+    // std::cout << __FILE__ << "time " << *(parameters_.time) << " [mu s]\n";
     return result;
 }
 

From e12958aebecc57c341317b40976191df3aedb5a0 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 6 Aug 2025 16:51:13 +0200
Subject: [PATCH 10/12] use optimizer as default

---
 .../OGL/DevicePersistent/ExecutorHandler.hpp   |  2 ++
 include/OGL/StoppingCriterion.hpp              | 18 +++++++-----------
 src/MatrixWrapper/Distributed.cpp              |  1 -
 test/unit/MatrixWrapper/Distributed.cpp        | 12 ++++++++++++
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index ec91a5db..4ee8382e 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -311,6 +311,8 @@ class ExecutorHandler
 
     label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
 
+    void set_ranks_per_gpu(label ranks_per_gpu) { device_id_handler_.ranks_per_gpu= ranks_per_gpu; }
+
     label get_owner_rank() const { return device_id_handler_.global_owner(); }
 
     const std::shared_ptr<gko::Executor> get_device_exec() const
diff --git a/include/OGL/StoppingCriterion.hpp b/include/OGL/StoppingCriterion.hpp
index b32cd791..944acd60 100644
--- a/include/OGL/StoppingCriterion.hpp
+++ b/include/OGL/StoppingCriterion.hpp
@@ -201,34 +201,30 @@ class StoppingCriterion {
                                   bool export_res, label prev_solve_iters,
                                   scalar prev_rel_cost) const
     {
+        word frequencyMode = "optimizer";
         label minIter = minIter_;
         label frequency = frequency_;
         // in case of export_res all residuals need to be computed
-        std::cout << __FILE__ << "adapt minIter and frequency0 \n";
         if (!export_res) {
-            std::cout << __FILE__ << "adapt minIter and frequency1 \n";
             if (prev_solve_iters > 0 && adapt_minIter_ && prev_rel_cost > 0) {
-                std::cout << __FILE__ << "adapt minIter and frequency2 \n";
                 minIter = prev_solve_iters * relaxationFactor_;
-                if (frequencyMode_ == "optimizer") {
+                if (frequencyMode == "optimizer") {
                     auto alpha = sqrt(
                         1.0 / (prev_solve_iters * (1.0 - relaxationFactor_)) *
                         prev_rel_cost);
                     frequency = min(norm_eval_limit_, max(1, label(1 / alpha)));
                 }
-                // if (frequencyMode_ == "relative") {
-                frequency = label(prev_solve_iters * 0.075) + 1;
-                // }
+                if (frequencyMode == "relative") {
+                    frequency = label(prev_solve_iters * 0.075) + 1;
+                }
             }
         }
 
         word msg = "Creating stopping criterion with minIter " +
                    std::to_string(minIter) + " frequency " +
                    std::to_string(frequency) + " prev_solve_iters " +
-                   std::to_string(prev_solve_iters) + +" adapt_minIter_  " +
-                   std::to_string(adapt_minIter_) + +" prev_rel_cost  " +
-                   std::to_string(prev_rel_cost) + " prev_solve_iters*0.075  " +
-                   std::to_string(prev_solve_iters * 0.075);
+                   std::to_string(prev_solve_iters) + " adapt_minIter_  " +
+                   std::to_string(adapt_minIter_) + " prev_rel_cost  ";
 
         MLOG_0(verbose, msg)
 
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index 94448ed3..f134d145 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -527,7 +527,6 @@ std::shared_ptr<RepartDistMatrix> create_impl(
     label rank = exec_handler.get_host_rank();
     auto exec = exec_handler.get_ref_exec();
     auto host_comm = *exec_handler.get_host_comm().get();
-    exec_handler.init_device_comm();
     auto device_comm = *exec_handler.get_device_comm().get();
     bool owner = repartitioner->is_owner(exec_handler);
 
diff --git a/test/unit/MatrixWrapper/Distributed.cpp b/test/unit/MatrixWrapper/Distributed.cpp
index 496a6c0c..272d5347 100644
--- a/test/unit/MatrixWrapper/Distributed.cpp
+++ b/test/unit/MatrixWrapper/Distributed.cpp
@@ -56,6 +56,7 @@ class Environment : public testing::Environment {
                            Foam::IOobject::MUST_READ),
             false);
 
+        // FIXME this needs the device_id_handler
         exec = std::make_shared<ExecutorHandler>(runTime_->thisDb(), dict,
                                                  "dummy", true);
 
@@ -172,6 +173,7 @@ TEST_P(DistMatL2D, canCreateDistributedMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
 
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
@@ -182,6 +184,7 @@ TEST_P(DistMatL2D, canCreateDistributedMatrix)
     gko::dim<2> global_vec_dim{repartitioner->get_orig_partition()->get_size(),
                                1};
     gko::dim<2> local_vec_dim{repartitioner->get_repart_dim()[0], 1};
+    exec.init_device_comm();
 
     auto distributed = create_distributed(exec, repartitioner, hostMatrix,
                                           matrix_format, fused, 0);
@@ -200,6 +203,8 @@ TEST_P(DistMatL2D, hasCorrectLocalMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto repartitioner = std::make_shared<Repartitioner>(
@@ -254,6 +259,8 @@ TEST_P(DistMatL2D, hasCorrectNonLocalMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto name = ((Environment *)global_env)->name_;
@@ -292,6 +299,8 @@ TEST_P(DistMatL2D, hasCorrectNonLocalMatrix)
 TEST_P(DistMatL2D, canApplyCorrectly)
 {
     auto [ranks_per_gpu, format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto name = ((Environment *)global_env)->name_;
@@ -314,12 +323,15 @@ TEST_P(DistMatL2D, canApplyCorrectly)
     x->fill(0);
 
     // Act
+    bool active = repartitioner->get_repart_size() != 0;
+    if (active){
     distributed->apply(b, x);
     auto res_x = std::vector<scalar>(
         x->get_local_vector()->get_const_values(),
         x->get_local_vector()->get_const_values() + local_vec_dim[0]);
 
     ASSERT_EQ(res_x, exp_x[name][fused][ranks_per_gpu][rank]);
+    }
 }
 
 int main(int argc, char *argv[])

From 28ee971b391c2687eaac29df783b2813731819fc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 7 Aug 2025 17:44:38 +0200
Subject: [PATCH 11/12] Avoid symmetric MPI_Igatherv

---
 include/OGL/.Preconditioner.hpp.swn           | Bin 16384 -> 0 bytes
 include/OGL/.Preconditioner.hpp.swo           | Bin 16384 -> 0 bytes
 include/OGL/CommunicationPattern.hpp          |   7 ++-
 .../OGL/DevicePersistent/ExecutorHandler.hpp  |   5 +-
 include/OGL/DevicePersistent/Vector.hpp       |  27 ++--------
 include/OGL/lduLduBase.hpp                    |   9 ----
 src/MatrixWrapper/Distributed.cpp             |  48 +++++++++++++-----
 src/StoppingCriterion.cpp                     |   4 --
 test/unit/MatrixWrapper/Distributed.cpp       |  12 ++---
 9 files changed, 57 insertions(+), 55 deletions(-)
 delete mode 100644 include/OGL/.Preconditioner.hpp.swn
 delete mode 100644 include/OGL/.Preconditioner.hpp.swo

diff --git a/include/OGL/.Preconditioner.hpp.swn b/include/OGL/.Preconditioner.hpp.swn
deleted file mode 100644
index 6793705a6f00ad2117ddacb5b1735306a1af1c01..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeHN-)|hn9iO(L1WG7sA9&$ma!BP)v3(Z@hiKziRqZ;swXsP;04Zph+nu}HcyD(*
zJ7=Fs)1pfJ2c@n0Pzk|PrAn>TAL1#9N9ao{KnNrxeh4ZdfdqYODeY%=X7_g2mj<Fr
zRjXO)<Gb7MkNM7LzVn@(^>jPu*EZ;hxp{)?HbVZ$&J7L^?t1+FwR;Hv-rb#J$CcpU
zUp<-3j!ztq=I6rLW1$}pSrAPz`GI!W_L*dDFZOvm?uYFl^1>0mZ=POnZwc<jksrt)
zj<}c`B*_#v_LgP9GH_iC93sc&k9L&losGlv>tB1|y1ZB;%YbFTGGH073|Iy%1C{~H
zfMwu+lL46=BtJyxUo_!&&F2?tpZ_%1Idk8u$v-vn4RikwHTge`{EWGOz%-Dz^NEq)
zYvku^^1mDT{YL(an*47@e!<8;tjWKWw`)GXYdV~_Z*JQU%YbFTGGH073|Iy%1C{~H
zfMvikU>UFs{9iJ_d_um7p1q?s!ubDxzW;yetAzXscm{YH_zCbNFa~yk`+&m$1r7m!
zy_JxUfwzDcfv12JI01Zeh>$-6zXYBD9sw4BTY=YZA>;+%5^xNd2U@^E;6~stUm@f@
z;5pz5@H612z-8e3fCP>JZ{AGEZ-J+P$AJalX5fRHU=O?tTmhZ{wtzc;R}K>Ld*Ii=
z7+3&q01g1p-bl#Lfro)xfRDaR$gcnfJPO<nym$j4KLSnyA0Hs(dEgn~X<z_s0e1nP
z;^o8Zz-z#7fF}VDSOz}Adx+lwd%$_%0pK{GUdX61dHX-`f`XPVHj*(-dFYCGH>IaY
zNklVQWF#~@%b|a?mBn(*M9R;=>||jT^k}mejJdCcQ4sZ8^b)e7cFo06)8{*Z$6Y?=
zUaK{6QPZSzyMY|Ig8L&+odR)t%#*Qbrg6B#?~BE7xz*a|Uis*6$BvWAIB}daJPrI2
z3zhm}iW4bJEICeh6oh{Bn#OCA3#M8wEObOeHg*HZ(k8wsCEexL%)V%}z9*D+z?xT)
zCzYcvP5d-n-K&O}bzBs|6yCDI^0F~;wGKUHNEk4|o7Xi6ldg6#{M&9CJjR<xN7aZx
zwi%^#J4n#DkQn!-6eC_$>|VS|*rm=QXu;A#PY0k=Pxq?Sk&X4u({87^!$mhvxjRFL
zX}{W{3Jq5(eaDfpn@XV~(=0v6gZ@BH&%ByJb%f~=+PDy1kP|m@(4a?1)>QRR%}*(5
zb0-zo^fS6CPrg>}N$~JoY-Z(WbhK}(nx0i=cg5FkEPO>XhN*Fy<4hKloDN5!3~0!A
zct}%ku*<|_jv}b$K66yCl;vx+_ot|j6JzoADA^P%yvN|S*{G;AW@yN|JZvs>>NR7E
zX;C!v)I6(7hA3Pd<+I1jPIM(K%1WsgmZp|Y8uWfjbCY}+NVK90O|vuCIa+Q;CCEPG
z#j~h#-e9TOsBs2AJr<_Cl%`oUHMM%M)d8FuZAGOZWkH2njL|5?bfL9slCkJG%l*8R
zPHMN-?|FIU$r7V40?fYE>Z)LD&2TQ&T_pYC6dzmtynI*fE)@Z1$x&LVS`^0zQ(4Hf
z`IxGD=(~v&i{^acYPP2AEm2d1XS5mhgKW)VVvl~qpA`1Gmu#u|Hr6q$rDBPe7PhO~
zWO3>GydFfG$&&g$WGGN$Z8WtJqq-|t;%4N!*{pNTtkY|Y<ijLX^Kwz{B|Kt7PHX+t
zBYf#{!3(8%5zVV<xsK`tSdJqp;xKjf+Kcs0|K7LaRn_;;XhX6&)Va*O0kRAkh-ot6
zi&75)?pBrE^6Ke#-KO6y**G-aOI=N(tNM*lt2au@baLsjC=YhE3wb)|>@rMgf()^y
zFIZWSZ8R&AwU%=qarYwk<Qd)%uo$Vd>dug4)C~jA(W9bWRGM<ll)P4)RXy!uDvK3=
z`y;CQnbsR<BqG>Bl9{L{WPbi{S)ip7kWD8(PVlre<dI~d<M@iLpgWQrrKFk?j&oKo
zIn|P)Gf5XDJW!YdsegaI>Y{CiYRS{jl%@b~s8vrDH(k8)+P6;+g|ZjMOqw2QJ#U(t
zc{DBi3uMu3_9$CCjaHOIp=4rP6V>9Z8x4xWT&Hl&TdaiVcQ58H>#E;jUfc~9{TNBh
zt995=a828cj%lXXw5$X<l;b=|J?F6~rKOI0AxqP|^qCoo&4?-tX~H~CS7SC*yOTT3
zk#pJ*yuY@oC&b*njgp|Zv%I>ZZk6t-1L_-D#eNX=80p$xnWU*Yt6p}XlvlDH&ldI!
z&_iaozr|9n6(?>N`$4qbkK6h~Nte)QB3OUOXdHQ*kanA%-CB8Qb{+etDCM(jKAZ);
zfD4D7-ddkMHrJVr#jIKb1e6Q?)ga`{ak3}WgYP*$mJXfo%->BBo7(FQVv!QH|Nk-e
z@6Q2h|BvlzcD;e$uL3U{*%c+Xmu0{*U>UFsSOzQumI2FvWxz6E8L$jk1}p>r6$UO-
iiq`|xcO2xhep@to-z&HriHP>ie{k?JT>tm+B>5MH00&|K

diff --git a/include/OGL/.Preconditioner.hpp.swo b/include/OGL/.Preconditioner.hpp.swo
deleted file mode 100644
index 59f5bb35988ed3c6ebd5b633ff6bf04be8b770a0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeHOTa4UR89wPPEHvd(5agkfyQv}%*`1kON`f=H5^UJ8G?yi9LZvn`9{bFUvmV>A
zJ-f3Ffxht42MVodLE?oZedz-Qo@jUgfp`Iml&F=siR%O6iH9I0AY8t48y|aSv)u?G
zA#CYedwl-$pZ|R4Kj--W?QJ*DEFCuv)%Ot`?;zx-uPydoeeSNOzjn_<gq}ap-2VYq
zaO<PD5;hMu57vXobpkKyJAOE^!l#A8hUa8X!;L)Jh`K?;58YsZ>yt;1HI@_VMxp0t
zeiYKA-izajimICx0~G^XV_=YVW)JN+y?@`{Cabt>`a$FVcb(pvj!L>>pkkn6pkkn6
zpkkn6pkkn6pkm;Ent?3dPCkPk+#!3hBG21O=YPrLjJ)43mH$(gFU$MiFO~m8mY<UM
z@0Sgzc5ce@M`SrImH%CqpOxibFO}aZ?|)F9cgPMY{qlBosu-vks2Hdis2Hdis2Hdi
zs2Hdis2Hdis2KRaWWezV`8ayMlf}z;|F7cz*WXUaHQ)+x8F&HsA}|EjfD^!jfC20P
z{(K)HH-O&(KLWlAq`)EI<_<!x11|%g1<nEw1NQ-M+)Kz0fK{Ll%mNPqcLH|+zkeHa
z0^bI%0?z}_0YhL7h=AR|FW*YYE5KKP3qTXN8~EcC>I1(8t^!X3j{xrle)ASWehyp$
zIsgS81`Yr_fxm1g<a@vXI0`HR4*=VMU))2;Gr&1u8n}5kA>RR>0X_-L0WIL2z>n`D
z<h#HN!1KU)AOx0x-N5TeUi<`j4frPTWuOmy2-pqWz}Q>|UIwlJmw`_KCjd5XzebYd
zSCbPRhCI^Dv#hk|B-FFxENS5@i!?L3E0JX#^TU(zJiG2YS>g|82&<FIipJJ!mYUKV
zSxcPQPTk&`lRRbeb~4(J11F<KD_f6g==7-}KLS~F+$>7g=iy^GF7m(ZN*%NFU5bxH
zP}dyUUaK7@s$8AxQD}N})pw~)ht#c&3#|EB&vqjxNvWMi!75G6!mk>cnyPc5V6UC_
zqbTb^q~ioB-9%eJS7~6z-M-y%Hf))HHlrGj#TPI|y@``rR(s$Fo|zXUG#ezL7(+g?
zCV`z(U;uwR?x=#WJYDg@e3WE<*fq0cKx^W@>}V3LrDjS4n_bj4q|6y)ks-Td%oW<6
zQnl+T%{bGLO$yAzvWqZoB-n)@`xeNoJ>j2)x<mEm`+W#qAXA+vF-+c?Ki53$w+eMq
z|0!zL4*Prdu#uSxh0&<UD=1NApOYlcdMj|+G?-_eVp*qXhbA<1=||uX%u3606UrQ^
zHY1e?okE3}7R=o;TQvPd^K9eLVd0p{&Dgoj+XQb^;$xyjm6Xq7oG@_*F8eHsc@iUb
zFP+_=JA+<{_c6_)nDsREy@3;)h?0IQ#n)L*GCy*R4O`S_i&zbG_3T<}Vku*D5QdHP
z3WNL3(DqT5ntQoFjkPBIFJVHyBUXcQJG$cw`SN6ubgYm0>^L_<ft+CS6SHEoV)BnV
zGBTO_Rbfd`&ZQ<^*>LUm*hr5rQMxo=XBc^Hbah2IT$+Jl*LIyWYjNraewgjwHxFvZ
zyE&$8eCiWTMxVs(a~t#>q$#3}x!msuDOM9hur^|ZFQAsS&{YnHi83W5JFb%I4}#3^
zCO&qhMSn=W1-|-*P$z567-GieWC;~md?x5v?ke{#*1It3X?YtfZ8C;hcMZNXZdi@8
zua=E@Lz^l2L@775`Chx(j>*}w`DSu^yXOUc`^5g%j<D64@<MkZF_sHam?cq=+Gg=L
zmPg%l@RYy@TZ!WX&}d~4pG=O>j)MU<r;C$q!Et*S;prIzeygi;Q9$jYu&qx`uWg%N
zTdQLk>3V95mDy%b4KTHMLlf3)7GvT0d?|I3bz^t23o~N;h4UE&f~3c&g+pjsixRJF
ziqa9a&rvr!MZ12Qv6j5{42cKrz;`V%g@g+3=c*}N?kbzvS(FnhCbC%2H*Qf;&!pZ!
zB2N5O?2giEjm<4zmgh8|PIMS!2k6r<a{|lq7+ZdOkWs7+MVGLwN5!LKag7k3B$fwk
z7+9c~UE;UX?+)#)tr?<XzpIuR;UJcETrF8*NfKfmcT5foUnhv1Y+jd}C?+>#UCz%&
z&GHR=w7V9Bl97;HY?qRJvCTX0kCxjqk4tW=V4l}Lr&N|jKjyem+i!UhJeL+%S4qL9
zbvYfAs%~1dU}QfVDkH_j$0n0%9d`pKO;z+Dr>Sv)u+<@r9hVx5k<-TwvP*t)ZcHb%
zyL3{_iuy;VwSvy-!r~EjOS{KDP(RK?q)#A>fzLLM<i5(jRo4|T;UhfiSjICD<lx?o
z4?8Ipg1OpO*AG{^QA3<qDI%U-apH9QjuGJ@m5@fmcy#&5<Fm&Q!-OfFUGkvJ@Ax#a
zjHAoPX7|^dvr#h3!l(qoBl$%?pbJsFp7`Bf_81*zma(t7?|=c@3_Q~Gq9i3O{$EFo
zx{er@#sBL4`74O&&jSa6ZNPQJ@z;QF03Cn=EVlmxV))CzCE!V*2K)?h`?J7fz(L?|
zh|j+TTmn81WPlAk09;3m{u0mzb^`AM-au^rCE!KiNnj7K9r!C^@;?DT1fB&>0Pg|b
z4ctIX{t|EncpTUP+zVVozpetW0xtqz06qt>{tkias2Hdis2Hdi_%AbX(J-*D6*p&c
zQ9S14(TYlP$xI)Tmm)tVA8XmOP8^BU4APoN3K4dT9Z)_<slk#qti-GtNL@Lrk?$FU
z5G9_m>LBlchZ+OfNEUhKajnqsrvIuu<j6YB44yiOBRsf-bj>I~5wHgW5z&h*tf`(x
z%-G3#GX~-w#4$v-#1DP+sPKPswvQaD$*AO^2M^t5SX?h2lb}XRf94Ev)5dJq<9i-a
zuxBoi7~=T>83-Q1m18YiSemP5DZtGuz*?ctW&uFtX|%B(BL?DKDTkFRmp3hIBc5z(
zRrRQ<s9#0~On2^K<G<J)>35Q6=0SPqSUEeiLe7y$Mvt6eyn^rrF?38NYC*@U8qK=$
zRdj~%7D|7%Ji@B}Mk8%(=uG1V{6~gV8FZUNt5wI_i#+PZ<drU@4Y3kt=gsEbyr~8g
zc~j)N7}J_dDOG9!M?;1Y#1yTMUSh}uhz$HYEuw0&^F@Q=jR67>{U*DX&W6*t&lDY1
z)tr1+mCnu&V&v7S%8#r%!GOOcVX?JX6DD*{)#fp|lZs{OG*={|o-<6%ya_Y)jFGB#
zi~i*z`IT6+MK)KMWKMc{lUK;EMA$6jNflP|)V|g~lFLuNDs=M}DG1T`;z*V?P9}Uq
z6ADdFU|7#_*T!d}-nvqSyoVSWomY{$xFxULqxJUS(L8_2S5Z5TkZ7ifmU))@)Wp8|
z9m8aA@r=1SV_K$arfV$X)=k#0H(CeGX{HAYJIhF!xvmY|mglWR%ahw;ma#Wm%(7<f
smJ?1lZ**OH`7=IEN0R!5J{HQyi7udD-@aFibx?FqWb@VReG_i|Z>mC^761SM

diff --git a/include/OGL/CommunicationPattern.hpp b/include/OGL/CommunicationPattern.hpp
index 64f1e221..3e7134c3 100644
--- a/include/OGL/CommunicationPattern.hpp
+++ b/include/OGL/CommunicationPattern.hpp
@@ -19,8 +19,13 @@ struct AllToAllPattern {
     std::vector<int> recv_offsets;
 };
 
-/* @brief computes AllToAllPattern for repart comm from global allToAll pattern
+/* @brief computes AllToAllPattern for repartioned communincator from global
+ * allToAll pattern by discarding all zero communication before and after the
+ * repartioner scope.
  *
+ * @param exec_handler The executor handler
+ * @param allToAll The original allToAll pattern
+ * @param start_rank the original comm_world rank
  */
 AllToAllPattern compute_repart_allToall(const ExecutorHandler &exec_handler,
                                         const AllToAllPattern allToAll,
diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index 4ee8382e..df56b32f 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -311,7 +311,10 @@ class ExecutorHandler
 
     label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
 
-    void set_ranks_per_gpu(label ranks_per_gpu) { device_id_handler_.ranks_per_gpu= ranks_per_gpu; }
+    void set_ranks_per_gpu(label ranks_per_gpu)
+    {
+        device_id_handler_.ranks_per_gpu = ranks_per_gpu;
+    }
 
     label get_owner_rank() const { return device_id_handler_.global_owner(); }
 
diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp
index 1b76273b..0691537c 100644
--- a/include/OGL/DevicePersistent/Vector.hpp
+++ b/include/OGL/DevicePersistent/Vector.hpp
@@ -171,7 +171,7 @@ class PersistentVector
 
     /** Copies the content of the distributed vector back to the original source
      **/
-    MPI_Request copy_back()
+    void copy_back()
     {
         auto exec = exec_.get_device_exec();
         auto rank = exec_.get_host_rank();
@@ -197,20 +197,12 @@ class PersistentVector
         // 	repartAllToAll.recv_counts[0] = 0;
         // }
 
+        // NOTE instead of all_to_all_v based communication MPI_Iscatterv
+        // seems to be prefarable
         // communicate_values(exec, ref_exec, comm, comm_pattern,
         //                    get_vector()->get_local_values(),
         //                    const_cast<T *>(memory_), host_buffer);
-        // std::cout << __FILE__
-        // 	<< " owner_rank " << exec_.get_owner_rank()
-        // 	<< " rank " << Pstream::myProcNo()
-        // 	<< " comm_pattern.send_counts: " << comm_pattern.send_counts
-        // 	<< " comm_pattern.recv_counts: " << comm_pattern.recv_counts
-        // 	<< " send_counts: " << repartAllToAll.send_counts
-        // 	<< " send_offsets: " << repartAllToAll.send_offsets
-        // 	<< " recv_counts: " << repartAllToAll.recv_counts
-        //        	<< "\n";
-
-        // auto start_rep = std::chrono::steady_clock::now();
+
         label send_size = comm_pattern.send_offsets.back();
         auto send_view = gko::array<scalar>::const_view(
             exec, send_size, get_vector()->get_local_values());
@@ -218,22 +210,13 @@ class PersistentVector
 
         tmp = send_view;
         tmp.set_executor(ref_exec);
-        // auto end_rep = std::chrono::steady_clock::now();
-        // auto delta_t_rep =
-        // std::chrono::duration_cast<std::chrono::microseconds>(end_rep -
-        // start_rep).count() /1000.0; std::cout << __FILE__ << " copy back: "
-        // << delta_t_rep << " [ms]\n";
-
 
         MPI_Request copy_back_req;
-        MPI_Iscatterv(tmp.get_data(),
-                      // get_vector()->get_local_values(),
-                      repartAllToAll.send_counts.data(),
+        MPI_Iscatterv(tmp.get_data(), repartAllToAll.send_counts.data(),
                       repartAllToAll.send_offsets.data(), MPI_DOUBLE,
                       const_cast<T *>(memory_), repartAllToAll.recv_counts[0],
                       MPI_DOUBLE, 0, repart_comm->get(), &copy_back_req);
         MPI_Wait(&copy_back_req, MPI_STATUS_IGNORE);
-        return copy_back_req;
     }
 
     /** Writes the content of the distributed vector to disk
diff --git a/include/OGL/lduLduBase.hpp b/include/OGL/lduLduBase.hpp
index 9bc5c0a8..3e997f77 100644
--- a/include/OGL/lduLduBase.hpp
+++ b/include/OGL/lduLduBase.hpp
@@ -306,15 +306,8 @@ class lduLduBase : public OGL_Info,
             delta_t_solve_ = delta_t_solve;
         }
 
-        // auto start_rep = std::chrono::steady_clock::now();
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
-        // auto copy_back_req = dist_x.copy_back();
-        // auto end_rep = std::chrono::steady_clock::now();
-        // auto delta_t_rep =
-        // std::chrono::duration_cast<std::chrono::microseconds>(end_rep -
-        // start_rep).count() /1000.0; std::cout << __FILE__ << " full copy
-        // back: " << delta_t_rep << " [ms]\n";
 
         auto bandwidth_copy_back =
             sizeof(scalar) * psi.size() / delta_t_copy_x_back / 1000.0;
@@ -346,8 +339,6 @@ class lduLduBase : public OGL_Info,
         std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
         MLOG_0(verbose_, msg)
 
-        // MPI_Wait(&copy_back_req,MPI_STATUS_IGNORE);
-
         return solverPerf;
     }
 
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index f134d145..6897cf7a 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -121,13 +121,15 @@ void generate_alltoall_update_data(
     std::vector<RepartDistMatrix::all_to_all_data> &update_data)
 {
     label linop_offset_store{0};
-    for (size_t i = 0; i < 3; i++) {
+    // NOTE in case of symmetric matrix 0 (upper) is same as 1 (lower)
+    // thus we can start at 1
+    label start = 0;
+    for (size_t i = start; i < 3; i++) {
         label interface_size = in->get_rows()[i].size();
         label linop_idx = (fuse) ? 0 : in->get_id()[i];
         label linop_offset = (fuse) ? linop_offset_store : 0;
         auto comm_pattern = compute_gather_to_owner_counts(
             exec_handler, ranks_per_owner, interface_size);
-
         size_t recv_size = comm_pattern.recv_offsets.back();
 
         // NOTE Probably dont need to store linops[linop-idx] because we can
@@ -395,6 +397,9 @@ void update_impl(
     auto all_to_all_update = [repart_comm, ref_exec, device_exec,
                               all_to_all_update_data, host_A, force_host_buffer,
                               exec_handler, rank]() {
+        // NOTE if symmetric (get it from host_A) we can skip id=0 and wait till
+        // id=1 has been copied to use device copy
+        //
         for (auto [id, comm_pattern, data_ptr] : all_to_all_update_data) {
             // auto start = std::chrono::steady_clock::now();
             auto repartAllToAll =
@@ -409,24 +414,43 @@ void update_impl(
             // communicate_values(ref_exec, device_exec, repart_comm,
             // repartAllToAll,
             //                    send_data_ptr, data_ptr, force_host_buffer);
-            // if ( repart_comm->rank() == 0 ) {
             // std::cout << __FILE__ <<
             //     " Pstream::rank " << Pstream::myProcNo() <<
             //     " repart_rank() "  << repart_comm->rank() <<
             //     " send_offsets.back() "  <<
+            //     " id " << id <<
             //     repartAllToAll.send_offsets.back() << " recv_counts: "  <<
             //     repartAllToAll.recv_counts << " recv_offsets: "  <<
             //     repartAllToAll.recv_offsets <<
             // std::endl;
-            // }
-            MPI_Request request;
-
-            MPI_Igatherv(send_data_ptr, repartAllToAll.send_offsets.back(),
-                         MPI_DOUBLE, data_ptr,
-                         repartAllToAll.recv_counts.data(),
-                         repartAllToAll.recv_offsets.data(), MPI_DOUBLE, 0,
-                         repart_comm->get(), &request);
-            MPI_Wait(&request, MPI_STATUS_IGNORE);
+
+            if (id == 0 && host_A->get_symmetric()) {
+            } else {
+                MPI_Request request;
+                MPI_Igatherv(send_data_ptr, repartAllToAll.send_offsets.back(),
+                             MPI_DOUBLE, data_ptr,
+                             repartAllToAll.recv_counts.data(),
+                             repartAllToAll.recv_offsets.data(), MPI_DOUBLE, 0,
+                             repart_comm->get(), &request);
+                MPI_Wait(&request, MPI_STATUS_IGNORE);
+            }
+
+            // Perform symmetric inter device copy
+            if (id == 1 && repart_comm->rank() == 0 &&
+                host_A->get_symmetric()) {
+                auto [zid, zcomm_pattern, zdata_ptr] =
+                    all_to_all_update_data[0];
+                // copy recv size data from data_ptr to zdata_ptr
+                //
+                label recv_buffer_size = repartAllToAll.recv_offsets.back();
+                auto l_view = gko::array<scalar>::view(
+                    device_exec, recv_buffer_size, data_ptr);
+
+                auto u_view = gko::array<scalar>::view(
+                    device_exec, recv_buffer_size, zdata_ptr);
+
+                u_view = l_view;
+            }
         }
     };
 
diff --git a/src/StoppingCriterion.cpp b/src/StoppingCriterion.cpp
index 76865bd2..07c7c71b 100644
--- a/src/StoppingCriterion.cpp
+++ b/src/StoppingCriterion.cpp
@@ -53,9 +53,6 @@ StoppingCriterion::OpenFOAMDistStoppingCriterion::compute_normfactor_dist(
                              end_axref - start_axref)
                              .count() /
                          1.0;
-    // std::cout << __FILE__ << " delta_t_axref " << delta_t_axref << " [mu
-    // s]\n";
-
     auto unity =
         gko::initialize<gko::matrix::Dense<scalar>>(1, {1.0}, device_exec);
 
@@ -177,7 +174,6 @@ bool StoppingCriterion::OpenFOAMDistStoppingCriterion::check_impl(
                               end_eval - start_eval)
                               .count() /
                           1.0;
-    // std::cout << __FILE__ << "time " << *(parameters_.time) << " [mu s]\n";
     return result;
 }
 
diff --git a/test/unit/MatrixWrapper/Distributed.cpp b/test/unit/MatrixWrapper/Distributed.cpp
index 272d5347..31b055f8 100644
--- a/test/unit/MatrixWrapper/Distributed.cpp
+++ b/test/unit/MatrixWrapper/Distributed.cpp
@@ -324,13 +324,13 @@ TEST_P(DistMatL2D, canApplyCorrectly)
 
     // Act
     bool active = repartitioner->get_repart_size() != 0;
-    if (active){
-    distributed->apply(b, x);
-    auto res_x = std::vector<scalar>(
-        x->get_local_vector()->get_const_values(),
-        x->get_local_vector()->get_const_values() + local_vec_dim[0]);
+    if (active) {
+        distributed->apply(b, x);
+        auto res_x = std::vector<scalar>(
+            x->get_local_vector()->get_const_values(),
+            x->get_local_vector()->get_const_values() + local_vec_dim[0]);
 
-    ASSERT_EQ(res_x, exp_x[name][fused][ranks_per_gpu][rank]);
+        ASSERT_EQ(res_x, exp_x[name][fused][ranks_per_gpu][rank]);
     }
 }
 

From 2506acd8ded24335d181e0b77fbf21934f705614 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 30 Oct 2025 14:35:46 +0100
Subject: [PATCH 12/12] fixup! typo

---
 include/OGL/DevicePersistent/Vector.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp
index 0691537c..c8610f59 100644
--- a/include/OGL/DevicePersistent/Vector.hpp
+++ b/include/OGL/DevicePersistent/Vector.hpp
@@ -198,7 +198,7 @@ class PersistentVector
         // }
 
         // NOTE instead of all_to_all_v based communication MPI_Iscatterv
-        // seems to be prefarable
+        // seems to be preferable
         // communicate_values(exec, ref_exec, comm, comm_pattern,
         //                    get_vector()->get_local_values(),
         //                    const_cast<T *>(memory_), host_buffer);