From d79c0f5271a76d23a9f285f00a7664e364eddb33 Mon Sep 17 00:00:00 2001 From: Khaled Sharafeldin <18222740+KhaledNabilSharafeldin@users.noreply.github.com> Date: Sun, 28 Jun 2026 10:05:11 -0500 Subject: [PATCH 1/2] Change atomic_add to use reinterpret_cast for gridval FFT gridval return `complex`, `atomicAdd` which is not supported on GPU (tested on LS40), result is just a freeze. A safe recast at final step to make sure type conforms. --- src/force_types/force_fft.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/force_types/force_fft.h b/src/force_types/force_fft.h index e3720d6..247d1e2 100644 --- a/src/force_types/force_fft.h +++ b/src/force_types/force_fft.h @@ -563,15 +563,15 @@ class ForceFFT : public Force { int ky = jb % Ngrid[1]; if (ky < 0) ky += Ngrid[1]; int kz = kb % Ngrid[2]; if (kz < 0) kz += Ngrid[2]; - Kokkos::atomic_add(&gridval[0](kx, ky, kz), W/Vs*b.x*t.x); - Kokkos::atomic_add(&gridval[1](kx, ky, kz), W/Vs*b.x*t.y); - Kokkos::atomic_add(&gridval[2](kx, ky, kz), W/Vs*b.x*t.z); - Kokkos::atomic_add(&gridval[3](kx, ky, kz), W/Vs*b.y*t.x); - Kokkos::atomic_add(&gridval[4](kx, ky, kz), W/Vs*b.y*t.y); - Kokkos::atomic_add(&gridval[5](kx, ky, kz), W/Vs*b.y*t.z); - Kokkos::atomic_add(&gridval[6](kx, ky, kz), W/Vs*b.z*t.x); - Kokkos::atomic_add(&gridval[7](kx, ky, kz), W/Vs*b.z*t.y); - Kokkos::atomic_add(&gridval[8](kx, ky, kz), W/Vs*b.z*t.z); + Kokkos::atomic_add(reinterpret_cast(&gridval[0](kx, ky, kz)), W/Vs*b.x*t.x); + Kokkos::atomic_add(reinterpret_cast(&gridval[1](kx, ky, kz)), W/Vs*b.x*t.y); + Kokkos::atomic_add(reinterpret_cast(&gridval[2](kx, ky, kz)), W/Vs*b.x*t.z); + Kokkos::atomic_add(reinterpret_cast(&gridval[3](kx, ky, kz)), W/Vs*b.y*t.x); + Kokkos::atomic_add(reinterpret_cast(&gridval[4](kx, ky, kz)), W/Vs*b.y*t.y); + Kokkos::atomic_add(reinterpret_cast(&gridval[5](kx, ky, kz)), W/Vs*b.y*t.z); + Kokkos::atomic_add(reinterpret_cast(&gridval[6](kx, ky, kz)), W/Vs*b.z*t.x); + Kokkos::atomic_add(reinterpret_cast(&gridval[7](kx, ky, kz)), W/Vs*b.z*t.y); + Kokkos::atomic_add(reinterpret_cast(&gridval[8](kx, ky, kz)), W/Vs*b.z*t.z); } } } From 84f925fdc0aaed656c41c82e10516d1bf0c1e67b Mon Sep 17 00:00:00 2001 From: Khaled Sharafeldin <18222740+KhaledNabilSharafeldin@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:03:28 -0500 Subject: [PATCH 2/2] Update atomic_add calls to use real() method Safer, same operation as `reinterpret_cast` which ignores the complex bits to minimize casting overhead. --- src/force_types/force_fft.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/force_types/force_fft.h b/src/force_types/force_fft.h index 247d1e2..33799f1 100644 --- a/src/force_types/force_fft.h +++ b/src/force_types/force_fft.h @@ -563,15 +563,15 @@ class ForceFFT : public Force { int ky = jb % Ngrid[1]; if (ky < 0) ky += Ngrid[1]; int kz = kb % Ngrid[2]; if (kz < 0) kz += Ngrid[2]; - Kokkos::atomic_add(reinterpret_cast(&gridval[0](kx, ky, kz)), W/Vs*b.x*t.x); - Kokkos::atomic_add(reinterpret_cast(&gridval[1](kx, ky, kz)), W/Vs*b.x*t.y); - Kokkos::atomic_add(reinterpret_cast(&gridval[2](kx, ky, kz)), W/Vs*b.x*t.z); - Kokkos::atomic_add(reinterpret_cast(&gridval[3](kx, ky, kz)), W/Vs*b.y*t.x); - Kokkos::atomic_add(reinterpret_cast(&gridval[4](kx, ky, kz)), W/Vs*b.y*t.y); - Kokkos::atomic_add(reinterpret_cast(&gridval[5](kx, ky, kz)), W/Vs*b.y*t.z); - Kokkos::atomic_add(reinterpret_cast(&gridval[6](kx, ky, kz)), W/Vs*b.z*t.x); - Kokkos::atomic_add(reinterpret_cast(&gridval[7](kx, ky, kz)), W/Vs*b.z*t.y); - Kokkos::atomic_add(reinterpret_cast(&gridval[8](kx, ky, kz)), W/Vs*b.z*t.z); + Kokkos::atomic_add(&gridval[0](kx, ky, kz).real(), W/Vs*b.x*t.x); + Kokkos::atomic_add(&gridval[1](kx, ky, kz).real(), W/Vs*b.x*t.y); + Kokkos::atomic_add(&gridval[2](kx, ky, kz).real(), W/Vs*b.x*t.z); + Kokkos::atomic_add(&gridval[3](kx, ky, kz).real(), W/Vs*b.y*t.x); + Kokkos::atomic_add(&gridval[4](kx, ky, kz).real(), W/Vs*b.y*t.y); + Kokkos::atomic_add(&gridval[5](kx, ky, kz).real(), W/Vs*b.y*t.z); + Kokkos::atomic_add(&gridval[6](kx, ky, kz).real(), W/Vs*b.z*t.x); + Kokkos::atomic_add(&gridval[7](kx, ky, kz).real(), W/Vs*b.z*t.y); + Kokkos::atomic_add(&gridval[8](kx, ky, kz).real(), W/Vs*b.z*t.z); } } }