From a01e8c826f3be5e739279a911cb158d0115fa361 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Wed, 5 Oct 2022 07:55:03 +0200 Subject: [PATCH 01/25] start merging with current QUDA develop --- include/global/PLEGMA_structs.h | 12 ++++++++---- include/global/PLEGMA_templates.h | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/include/global/PLEGMA_structs.h b/include/global/PLEGMA_structs.h index 314938f1..6542c86c 100644 --- a/include/global/PLEGMA_structs.h +++ b/include/global/PLEGMA_structs.h @@ -63,14 +63,18 @@ struct pointer_holder { void copyToDeviceConstant() { if(devPointer != nullptr) { - cudaMemcpyToSymbol( *devPointer, hostPointer, bytes*size); - checkCudaError(); + cudaError_t err = cudaMemcpyToSymbol( *devPointer, hostPointer, bytes*size); + if (err != cudaSuccess) { + errorQuda("Failed to copy constant host memory of size to device %zu \n", size); + } } } void copyFromDeviceConstant() { if(false and devPointer != nullptr) { - cudaMemcpyFromSymbol(hostPointer, *devPointer, bytes*size, 0, cudaMemcpyDeviceToHost); - checkCudaError(); + cudaError_t err = cudaMemcpyFromSymbol(hostPointer, *devPointer, bytes*size, 0, cudaMemcpyDeviceToHost); + if (err != cudaSuccess) { + errorQuda("Failed to copy constant host memory of size to device %zu \n", size); + } } } bool checkDeviceConstant() { diff --git a/include/global/PLEGMA_templates.h b/include/global/PLEGMA_templates.h index 9f548309..60a8c9b0 100644 --- a/include/global/PLEGMA_templates.h +++ b/include/global/PLEGMA_templates.h @@ -46,14 +46,18 @@ template inline void hostFree(T &ptr) { // Pinned memory allocation template inline void hostMallocPinned(T &ptr, size_t size){ - cudaMallocHost((void**)&ptr, size); - checkCudaError(); + cudaError_t err = cudaMallocHost((void**)&ptr, size); + if (err != cudaSuccess) { + errorQuda("Failed to allocate host memory of size %zu \n", size); + } HGC_used_memory += size; } template inline void hostFreePinned(T &ptr, size_t size) { - cudaFreeHost(ptr); - checkCudaError(); + cudaError_t err = cudaFreeHost(ptr); + if (err != cudaSuccess) { + errorQuda("Failed to free host memory of size %zu \n", size); + } ptr=NULL; HGC_used_memory -= size; } From f31adba412b26d9d849f35b9a9b723173f8e9065 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Mon, 10 Oct 2022 08:42:10 +0200 Subject: [PATCH 02/25] small progress in the compilation of current develop QUDA --- CMakeLists.txt | 18 ++++++++-------- include/PLEGMA_Hprobing.h | 8 +++++--- include/PLEGMA_utils.h | 1 + include/global/PLEGMA_Options.h | 1 + include/global/PLEGMA_prints.hpp | 6 +++--- include/io/PLEGMA_lime.h | 2 +- include/utils/QUDA_interface.h | 8 ++++---- include/utils/QUDA_types.h | 24 +++++++++++----------- lib/PLEGMA_Field.cu | 7 ++++--- lib/kernels/PLEGMA_kernel_tuner.cuh | 32 +++++++++++++++++------------ utils/PLEGMA_utils.cpp | 10 +++++++++ 11 files changed, 69 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ec5da76..b771bf05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON) set(CMAKE_COLOR_MAKEFILE ON) set(CMAKE_CUDA_SEPARABLE_COMPILATION ON) set(CMAKE_CXX_STANDARD_REQUIRED True) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) include(CheckIncludeFileCXX) include(CheckSymbolExists) @@ -264,7 +264,7 @@ find_package(PythonInterp) if(${CMAKE_VERSION} VERSION_GREATER 3.7.99) find_package(CUDAWrapper) set(USING_CUDA_LANG_SUPPORT True) - set(CMAKE_CUDA_STANDARD 14) + set(CMAKE_CUDA_STANDARD 17) set(CMAKE_CUDA_STANDARD_REQUIRED True) else() set(CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE FILEPATH "Host side compiler used by NVCC") @@ -415,7 +415,7 @@ add_definitions(-D__COMPUTE_CAPABILITY__=${COMP_CAP}) # NVCC FLAGS independet off build type if(NOT USING_CUDA_LANG_SUPPORT) - set(CUDA_NVCC_FLAGS "-std c++11 -arch=${GPU_ARCH} -ftz=true -prec-div=false -prec-sqrt=false") + set(CUDA_NVCC_FLAGS "-std c++17 -arch=${GPU_ARCH} -ftz=true -prec-div=false -prec-sqrt=false") else() set(CUDA_NVCC_FLAGS "-ftz=true -prec-div=false -prec-sqrt=false") set(CMAKE_CUDA_FLAGS "-arch=${GPU_ARCH}" CACHE STRING "Flags used by the CUDA compiler" FORCE) @@ -471,17 +471,17 @@ set(CMAKE_CUDA_FLAGS_DEBUG "${CUDA_NVCC_FLAGS} -g -DHOST_DEBUG -G" CACHE STRING FORCE ) #define CXX FLAGS -set(CMAKE_CXX_FLAGS_DEVEL "${OpenMP_CXX_FLAGS} -O3 -Wall ${CLANG_FORCE_COLOR}" CACHE STRING +set(CMAKE_CXX_FLAGS_DEVEL "${OpenMP_CXX_FLAGS} -std=c++17 -O3 -Wall ${CLANG_FORCE_COLOR}" CACHE STRING "Flags used by the C++ compiler during regular development builds.") -set(CMAKE_CXX_FLAGS_STRICT "${OpenMP_CXX_FLAGS} -O3 -Wall -Werror ${CLANG_NOERROR}" CACHE STRING +set(CMAKE_CXX_FLAGS_STRICT "${OpenMP_CXX_FLAGS} -std=c++17 -O3 -Wall -Werror ${CLANG_NOERROR}" CACHE STRING "Flags used by the C++ compiler during strict jenkins builds.") -set(CMAKE_CXX_FLAGS_RELEASE "${OpenMP_CXX_FLAGS} -O3 -w" CACHE STRING +set(CMAKE_CXX_FLAGS_RELEASE "${OpenMP_CXX_FLAGS} -std=c++17 -O3 -w" CACHE STRING "Flags used by the C++ compiler during release builds.") -set(CMAKE_CXX_FLAGS_HOSTDEBUG "${OpenMP_CXX_FLAGS} -Wall -Wno-unknown-pragmas -g -fno-inline -DHOST_DEBUG ${CLANG_FORCE_COLOR}" CACHE STRING +set(CMAKE_CXX_FLAGS_HOSTDEBUG "${OpenMP_CXX_FLAGS} -std=c++17 -Wall -Wno-unknown-pragmas -g -fno-inline -DHOST_DEBUG ${CLANG_FORCE_COLOR}" CACHE STRING "Flags used by the C++ compiler during host-debug builds.") -set(CMAKE_CXX_FLAGS_DEVICEDEBUG "${OpenMP_CXX_FLAGS} -Wall -Wno-unknown-pragmas ${CLANG_FORCE_COLOR}" CACHE STRING +set(CMAKE_CXX_FLAGS_DEVICEDEBUG "${OpenMP_CXX_FLAGS} -std=c++17 -Wall -Wno-unknown-pragmas ${CLANG_FORCE_COLOR}" CACHE STRING "Flags used by the C++ compiler during device-debug builds.") -set(CMAKE_CXX_FLAGS_DEBUG "${OpenMP_CXX_FLAGS} -Wall -Wno-unknown-pragmas -g -fno-inline -DHOST_DEBUG ${CLANG_FORCE_COLOR}" CACHE STRING +set(CMAKE_CXX_FLAGS_DEBUG "${OpenMP_CXX_FLAGS} -std=c++17 -Wall -Wno-unknown-pragmas -g -fno-inline -DHOST_DEBUG ${CLANG_FORCE_COLOR}" CACHE STRING "Flags used by the C++ compiler during full (host+device) debug builds.") OPTION(USE_OpenMP "Use OpenMP" ON) diff --git a/include/PLEGMA_Hprobing.h b/include/PLEGMA_Hprobing.h index 08c50c3b..71c69c7e 100644 --- a/include/PLEGMA_Hprobing.h +++ b/include/PLEGMA_Hprobing.h @@ -1,4 +1,6 @@ #pragma once +//#include +#include namespace plegma { inline int getVecToInd(std::vector x, std::vector L){ @@ -100,15 +102,15 @@ namespace plegma { createColLattice(); // if(check)checkColoring(); cudaMalloc((void**)&d_arrVc, HGC_localVolume*sizeof(int)); - checkCudaError(); +// checkQudaError(); cudaMemcpy(d_arrVc, h_arrVc, HGC_localVolume*sizeof(int), cudaMemcpyHostToDevice); - checkCudaError(); +// checkQudaError(); } ~PLEGMA_Hprobing(){ delete[] h_arrVc; delete[] arrlc; cudaFree(d_arrVc); - checkCudaError(); + //checkQudaError(); } int* H_arrVc() const{return h_arrVc;} int* D_arrVc() const{return d_arrVc;} diff --git a/include/PLEGMA_utils.h b/include/PLEGMA_utils.h index bfe43de1..b4abaf18 100644 --- a/include/PLEGMA_utils.h +++ b/include/PLEGMA_utils.h @@ -33,6 +33,7 @@ void setInvertParam(QudaInvertParam &inv_param); void setEigMultigridParam(QudaMultigridParam &mg_param, QudaEigParam *mg_eig_param); #endif //============== PLEGMA_utils.cpp =======================================// +void checkQudaError(); void createMom(int *Nmom, int momElem[][3], int Q_qs); extern const std::vector listAvailOptPLEGMA; void initializeOptions(int argc, char **argv, bool withQuda=true, std::vector listOptPLEGMA = listAvailOptPLEGMA); diff --git a/include/global/PLEGMA_Options.h b/include/global/PLEGMA_Options.h index 2be61241..4c7a08dc 100644 --- a/include/global/PLEGMA_Options.h +++ b/include/global/PLEGMA_Options.h @@ -1,5 +1,6 @@ #pragma once #include "PLEGMA_templates.h" +#include struct argument{std::string name, value;}; class Arguments{ diff --git a/include/global/PLEGMA_prints.hpp b/include/global/PLEGMA_prints.hpp index 6e13c588..86a8b07c 100644 --- a/include/global/PLEGMA_prints.hpp +++ b/include/global/PLEGMA_prints.hpp @@ -3,12 +3,12 @@ * inspired by QUDA. */ #pragma once - +#include extern bool HGC_hold_exit; // used to hold exit until all the errors have been printed extern bool HGC_init_PLEGMA_flag; extern struct global_vars HGC_global_vars; extern class Options * HGC_options; - +using namespace quda; #define PLEGMA_exit(value) do { \ if(! HGC_hold_exit) { \ if (HGC_init_PLEGMA_flag) { \ @@ -53,7 +53,7 @@ extern class Options * HGC_options; fprintf(getOutputFile(), "%sERROR: ", getOutputPrefix()); \ fprintf(getOutputFile(), __VA_ARGS__); \ fprintf(getOutputFile(), " (rank %d, host %s, " __FILE__ ":%d in %s())\n", \ - comm_rank(), comm_hostname(), __LINE__, __func__); \ + comm_rank(), comm_hostname(), __LINE__, __func__); \ fprintf(getOutputFile(), "%s last kernel called was (name=%s,volume=%s,aux=%s)\n", \ getOutputPrefix(), getLastTuneKey().name, \ getLastTuneKey().volume, getLastTuneKey().aux); \ diff --git a/include/io/PLEGMA_lime.h b/include/io/PLEGMA_lime.h index 1015438a..4d70a5a2 100644 --- a/include/io/PLEGMA_lime.h +++ b/include/io/PLEGMA_lime.h @@ -2,7 +2,7 @@ extern "C" { #include } - +#include #include #include #include diff --git a/include/utils/QUDA_interface.h b/include/utils/QUDA_interface.h index 6a74e3ae..296dc016 100644 --- a/include/utils/QUDA_interface.h +++ b/include/utils/QUDA_interface.h @@ -17,7 +17,7 @@ namespace quda { QudaMultigridParam mg_param; Dirac *D, *DSloppy, *DPre; DiracMatrix *M, *MSloppy, *MPre; - cudaColorSpinorField *b, *x; + ColorSpinorField *b, *x; #ifdef QUDA_INCLUDES_COMMIT_775a033 QudaEigParam *mg_eig_param; #endif @@ -28,9 +28,9 @@ namespace quda { void UpdateSolver(); QUDA_solver(double mu); virtual ~QUDA_solver(); - cudaColorSpinorField* solve(cudaColorSpinorField * rhs); + ColorSpinorField* solve(ColorSpinorField * rhs); template - cudaColorSpinorField* solve(PLEGMA_Vector &vectorIn); + ColorSpinorField* solve(PLEGMA_Vector &vectorIn); template void solve(PLEGMA_Vector &out, PLEGMA_Vector &in); template @@ -43,7 +43,7 @@ namespace quda { DiracParam dParam; QudaInvertParam inv_param; Dirac *D; - cudaColorSpinorField *in, *out; + ColorSpinorField *in, *out; template void apply(); public: //only QUDA_WILSON_DSLASH, QUDA_CLOVER_WILSON_DSLASH, QUDA_TWISTED_MASS_DSLASH, QUDA_TWISTED_CLOVER_DSLASH diff --git a/include/utils/QUDA_types.h b/include/utils/QUDA_types.h index b86b2ad1..f824de87 100644 --- a/include/utils/QUDA_types.h +++ b/include/utils/QUDA_types.h @@ -284,8 +284,8 @@ inline QudaSchwarzType get_schwarz_type(std::string s) { inline QudaTwistFlavorType get_flavor_type(std::string s) { if(s=="singlet") return QUDA_TWIST_SINGLET; - else if(s=="deg-doublet") - return QUDA_TWIST_DEG_DOUBLET; +// else if(s=="deg-doublet") +// return QUDA_TWIST_DEG_DOUBLET; else if(s=="nondeg-doublet") return QUDA_TWIST_NONDEG_DOUBLET; else if(s=="no") @@ -300,8 +300,8 @@ inline std::string get_flavor_str(QudaTwistFlavorType type) { switch(type) { case QUDA_TWIST_SINGLET: return "singlet"; - case QUDA_TWIST_DEG_DOUBLET: - return "deg-doublet"; +// case QUDA_TWIST_DEG_DOUBLET: +// return "deg-doublet"; case QUDA_TWIST_NONDEG_DOUBLET: return "nondeg-doublet"; case QUDA_TWIST_NO: @@ -321,10 +321,10 @@ inline QudaInverterType get_solver_type(std::string s) { return QUDA_GCR_INVERTER; else if(s=="pcg") return QUDA_PCG_INVERTER; - else if(s=="mpcg") - return QUDA_MPCG_INVERTER; - else if(s=="mpbicgstab") - return QUDA_MPBICGSTAB_INVERTER; +// else if(s=="mpcg") +// return QUDA_MPCG_INVERTER; +// else if(s=="mpbicgstab") +// return QUDA_MPBICGSTAB_INVERTER; else if(s=="mr") return QUDA_MR_INVERTER; else if(s=="sd") @@ -377,10 +377,10 @@ inline std::string get_solver_str(QudaInverterType type) { return "gcr"; case QUDA_PCG_INVERTER: return "pcg"; - case QUDA_MPCG_INVERTER: - return "mpcg"; - case QUDA_MPBICGSTAB_INVERTER: - return "mpbicgstab"; +// case QUDA_MPCG_INVERTER: +// return "mpcg"; +// case QUDA_MPBICGSTAB_INVERTER: +// return "mpbicgstab"; case QUDA_MR_INVERTER: return "mr"; case QUDA_SD_INVERTER: diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index f383fb95..cdf8ee4f 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -741,7 +742,7 @@ void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign) Float2 *x; cudaMalloc((void**)&x, V*2*sizeof(Float)); cudaMemset((void*) x,0,V*2*sizeof(Float)); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); std::vector momF(mom.begin(), mom.end()); createMomField(x, momF, D3D4, sign); for(int dof = 0; dof < field_length; dof++) @@ -798,7 +799,7 @@ static void cudaCopyOrCast(PLEGMA_Field &fieldOut, PLEGMA_Field @@ -1082,7 +1083,7 @@ void PLEGMA_Field3D::absorb(const PLEGMA_Field &field, int global_ cudaMemset(pointer_dst, 0, V3 * sizeof(Float)); } } - checkCudaError(); + checkQudaError(); } template diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index 31829ee7..e29e070e 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -1,5 +1,8 @@ #include #include +#include +#include +#include using namespace quda; #ifndef PLEGMA_KERNEL_TUNER_H @@ -7,6 +10,7 @@ using namespace quda; #define THREADS_PER_BLOCK 64 + extern __device__ cudaDeviceProp devProp; // struct that contains all variables @@ -167,16 +171,16 @@ protected: // launching utilities template - void callKernel(TuneParam tp, const cudaStream_t stream, seq) { + void callKernel(TuneParam tp, const qudaStream_t stream, seq) { if( typeid(ProfileStruct &)==typeid(std::get<0>(args))) { // in case ProfileStruct is the first argument we call it as a function (*kernel)(std::get(args)...); } else { - (*kernel)<<>>(std::get(args)...); + (*kernel)<<>>(std::get(args)...); } - cudaDeviceSynchronize(); + // cudaDeviceSynchronize(); } - void launchKernel( TuneParam tp, const cudaStream_t stream) { + void launchKernel( TuneParam tp, const qudaStream_t stream) { callKernel(tp,stream,typename gens::type()); } @@ -203,7 +207,7 @@ public: // tuning functions void tune(); void run(); - void apply(const cudaStream_t &stream); + void apply(const qudaStream_t &stream); void apply(); // utilities @@ -229,7 +233,7 @@ void PLEGMA_kernel_tuner::tune(){ // apply tuning and/or running with/without tuning template -void PLEGMA_kernel_tuner::apply(const cudaStream_t &stream){ +void PLEGMA_kernel_tuner::apply(const qudaStream_t &stream){ #ifdef PLEGMA_NO_TUNING // asked for no tuning, using default parameters tune(); @@ -238,7 +242,7 @@ void PLEGMA_kernel_tuner::apply(const cudaStream_t &stream){ // performing tuning if we need to tune if( !ps.tuned && !activeTuning() && ps.tune_globally ) comm_barrier(); //syncronizing if( !ps.tuned ) ps.tp = tuneLaunch(*this, getTuning(), (QudaVerbosity) HGC_verbosity); - if( !ps.tuned ) cudaGetLastError(); // ensuring that the error state has been clean + if( !ps.tuned ) qudaGetLastError(); // ensuring that the error state has been clean if( !activeTuning() ) ps.tuned = true; if( onlyTuning && !activeTuning() ) return; @@ -247,14 +251,16 @@ void PLEGMA_kernel_tuner::apply(const cudaStream_t &stream){ // HACK: For unknown reason, the Out Of Memory error state is not seen in QUDA/lib/tune.cpp // by error = cudaGetLastError(); (line 765). // So here we use jitify_error to communicate to the tuner the failure of the kernel. - cudaError_t error = cudaPeekAtLastError(); + qudaError_t error = qudaGetLastError(); if( activeTuning() && ps.tune_globally ) { - double tmp = error; - comm_allreduce_max(&tmp); - error = (cudaError_t) tmp; + //double tmp = error; + std::vector tmp; + tmp.push_back(error); + quda::comm_allreduce_max(tmp); + error = (qudaError_t) tmp[0]; } - if( error != cudaSuccess ) jitify_error = (CUresult) error; - if( !activeTuning() ) checkCudaError(); + //if( error != cudaSuccess ) jitify_error = (CUresult) error; + if( !activeTuning() ) checkQudaError(); #endif } diff --git a/utils/PLEGMA_utils.cpp b/utils/PLEGMA_utils.cpp index 55fed85a..c7ff2094 100644 --- a/utils/PLEGMA_utils.cpp +++ b/utils/PLEGMA_utils.cpp @@ -38,6 +38,16 @@ void initializeOptions(int argc, char **argv, bool withQuda, std::vector& listOpt, std::function add_options){ PLEGMA_printf("Reading options from file %s\n", filename.c_str()); if(!filename.empty() and access( filename.c_str(), F_OK ) != -1){ From 543f38513b66fd7f84a7c0b25fea3751c015bcca Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Mon, 10 Oct 2022 08:49:11 +0200 Subject: [PATCH 03/25] include target cuda --- lib/kernels/PLEGMA_kernel_tuner.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index e29e070e..7c9ba868 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -176,7 +176,7 @@ protected: // in case ProfileStruct is the first argument we call it as a function (*kernel)(std::get(args)...); } else { - (*kernel)<<>>(std::get(args)...); + (*kernel)<<>>(std::get(args)...); } // cudaDeviceSynchronize(); } From 7bf7678eba27b954d7741eeeedbf6c0fec4f9510 Mon Sep 17 00:00:00 2001 From: Jacopo Tarello Date: Thu, 13 Oct 2022 12:18:30 +0300 Subject: [PATCH 04/25] added codes and kernels for Soft Function and TMDPDFs with one-end and wall-source methods --- .#WallSourceNucleon.shexit | 1 + .#slurm-243497.out | 1 + CMakeLists.txt | 5 +- include/PLEGMA_Correlator.h | 27 + include/PLEGMA_Field.h | 2 + include/PLEGMA_Propagator.h | 1 + lib/CMakeLists.txt | 1 + lib/PLEGMA_Correlator.cu | 123 +++++ lib/PLEGMA_Field.cu | 5 + lib/PLEGMA_Propagator.cu | 12 + lib/kernels/PLEGMA_QWF.cuh | 346 +++++++++++++ lib/kernels/PLEGMA_baryons.cuh | 182 +++++++ lib/kernels/PLEGMA_baryons_NtoN.cu | 50 ++ lib/kernels/PLEGMA_field_utils.cuh | 18 + lib/kernels/PLEGMA_mesons.cuh | 349 +++++++++++++ lib/kernels/PLEGMA_mesonsNew.cuh | 157 ++++++ lib/kernels/PLEGMA_propagator_utils.cuh | 41 +- lib/kernels/PLEGMA_threep.cuh | 4 +- lib/kernels/PLEGMA_threep_staple.cu | 201 ++++++++ plegma/CMakeLists.txt | 20 +- plegma/SoftFunction.cpp | 461 +++++++++++++++++ plegma/SoftFunctionWall.cpp | 406 +++++++++++++++ plegma/TMDPDFs_nucleon.cpp | 630 ++++++++++++++++++++++++ plegma/TMDPDFs_pion.cpp | 517 +++++++++++++++++++ plegma/WallSourceNucleon.cpp | 145 ++++++ plegma/Zfac.cpp | 24 +- plegma/ZfacStaple.cpp | 190 +++++++ 27 files changed, 3899 insertions(+), 20 deletions(-) create mode 120000 .#WallSourceNucleon.shexit create mode 120000 .#slurm-243497.out create mode 100644 lib/kernels/PLEGMA_QWF.cuh create mode 100644 lib/kernels/PLEGMA_mesonsNew.cuh create mode 100644 lib/kernels/PLEGMA_threep_staple.cu create mode 100644 plegma/SoftFunction.cpp create mode 100644 plegma/SoftFunctionWall.cpp create mode 100644 plegma/TMDPDFs_nucleon.cpp create mode 100644 plegma/TMDPDFs_pion.cpp create mode 100644 plegma/WallSourceNucleon.cpp create mode 100644 plegma/ZfacStaple.cpp diff --git a/.#WallSourceNucleon.shexit b/.#WallSourceNucleon.shexit new file mode 120000 index 00000000..b3f13a0a --- /dev/null +++ b/.#WallSourceNucleon.shexit @@ -0,0 +1 @@ +jtarello@cyclamen.4608:1636095945 \ No newline at end of file diff --git a/.#slurm-243497.out b/.#slurm-243497.out new file mode 120000 index 00000000..a09050b6 --- /dev/null +++ b/.#slurm-243497.out @@ -0,0 +1 @@ +jtarello@cyclamen.21678:1651238169 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0215e758..38145a18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,11 +149,11 @@ set(PLEGMA_LIGHT_BARYONS FALSE CACHE BOOL "compile contractions for all light ba if(PLEGMA_LIGHT_BARYONS) add_definitions(-DPLEGMA_LIGHT_BARYONS) endif() -set(PLEGMA_SCATTERING_CONTRACTIONS FALSE CACHE BOOL "compile contractions for pi-N scattering I=3/2") +set(PLEGMA_SCATTERING_CONTRACTIONS TRUE CACHE BOOL "compile contractions for pi-N scattering I=3/2") if (PLEGMA_SCATTERING_CONTRACTIONS) add_definitions(-DPLEGMA_SCATTERING_CONTRACTIONS) endif() -set(PLEGMA_UDSC_BARYONS FALSE CACHE BOOL "compile contractions for all u,d,s and c baryons") +set(PLEGMA_UDSC_BARYONS TRUE CACHE BOOL "compile contractions for all u,d,s and c baryons") if(PLEGMA_UDSC_BARYONS) add_definitions(-DPLEGMA_UDSC_BARYONS) endif() @@ -546,4 +546,3 @@ add_subdirectory(tests) ### add PLEGMA directory add_subdirectory(utils) add_subdirectory(plegma) - diff --git a/include/PLEGMA_Correlator.h b/include/PLEGMA_Correlator.h index f4e57c3e..53d12711 100644 --- a/include/PLEGMA_Correlator.h +++ b/include/PLEGMA_Correlator.h @@ -193,9 +193,17 @@ namespace plegma { } void contractMesons(PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2); + + void contractMesonsNew(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2); void contractBaryons(PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2); + + void contractBaryonsWall(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, + PLEGMA_Propagator &prop3, + PLEGMA_Propagator &prop4); void contractBaryonsUDSC(PLEGMA_Propagator &propUP, PLEGMA_Propagator &propDN, @@ -231,6 +239,25 @@ namespace plegma { PLEGMA_Su3field &su3, int signProps, std::vector gammas); + void contractNucleonThrp_staple(PLEGMA_Propagator &bwdProp, + PLEGMA_Propagator &fwdProp, + PLEGMA_Su3field &su3, + int signProps, std::vector gammas, bool isZfac, int b, int l, int z); + + void contractMesonsFourp_ultralocal_oneendtrick(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2,int b); + + void contractMesonsFourp_ultralocal(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, + PLEGMA_Propagator &prop3, + PLEGMA_Propagator &prop4,int b); + + void contractTMDWFMesonsTrick_Zfac(PLEGMA_Propagator &prop1, + PLEGMA_Su3field &staple, int b, int l); + + void contractTMDWFMesons_Zfac(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, + PLEGMA_Su3field &staple, int b, int l); virtual void writeASCII(std::string filename) const; virtual void writeHDF5(std::string filename) const; diff --git a/include/PLEGMA_Field.h b/include/PLEGMA_Field.h index fcb81106..b3da1e38 100644 --- a/include/PLEGMA_Field.h +++ b/include/PLEGMA_Field.h @@ -159,6 +159,8 @@ namespace plegma { void trPmunu(PLEGMA_Gauge &gauge, std::pair munu); + void SU3Trace(PLEGMA_Su3field &su3field); + /** @brief Absorbs all elements from a 3D field and puts it at a specific global time of the 4D field @param PLEGMA_Field3D prop, The 3D field diff --git a/include/PLEGMA_Propagator.h b/include/PLEGMA_Propagator.h index 56bb7aa8..5eaa1f99 100644 --- a/include/PLEGMA_Propagator.h +++ b/include/PLEGMA_Propagator.h @@ -57,6 +57,7 @@ namespace plegma { void applyBoundaries_device(int t0); void rotateToPhysicalBase_host(int sign); void rotateToPhysicalBase_device(int sign); + void PropmulVVdag(PLEGMA_Vector &vec1,PLEGMA_Vector &vec2); }; /////////////////////////////// diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index cd4429f6..d787bf6c 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -18,6 +18,7 @@ set (PLEGMA_OBJS kernels/PLEGMA_threep_threeD_part3.cu kernels/PLEGMA_threep_threeD_part4.cu kernels/PLEGMA_threep_wilsonLine.cu + kernels/PLEGMA_threep_staple.cu PLEGMA_QLoops.cu PLEGMA_FT.cu PLEGMA_BLAS.cu diff --git a/lib/PLEGMA_Correlator.cu b/lib/PLEGMA_Correlator.cu index d9034ed6..034770ec 100644 --- a/lib/PLEGMA_Correlator.cu +++ b/lib/PLEGMA_Correlator.cu @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -48,6 +50,19 @@ contractMesons(PLEGMA_Propagator &prop1, contract_mesons(prop1,prop2,*this); } +template +void PLEGMA_Correlator:: +contractMesonsNew(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2 ){ + + shape = {10}; + datasets = {"twop_meson"}; + groups = {"mesons"}; + description = "pseudoscalar, scalar, g5g1, g5g2, g5g3, g5g4, g1, g2, g3, g4"; + + initialize(); + contract_mesons_new(prop1,prop2,*this); +} template void PLEGMA_Correlator:: @@ -69,6 +84,29 @@ contractBaryons(PLEGMA_Propagator &prop1, contract_baryons(prop1,prop2,*this); } +template +void PLEGMA_Correlator:: +contractBaryonsWall(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, + PLEGMA_Propagator &prop3, + PLEGMA_Propagator &prop4){ + + shape = {16}; + datasets = {"twop_baryon_1", "twop_baryon_2"}; + groups = {"baryons/nucl_nucl", +#ifdef PLEGMA_LIGHT_BARYONS + "baryons/nucl_nucl2","baryons/nucl2_nucl","baryons/nucl2_nucl2", + "baryons/deltap_deltaz_11","baryons/deltap_deltaz_22","baryons/deltap_deltaz_33", + "baryons/deltapp_deltamm_11","baryons/deltapp_deltamm_22","baryons/deltapp_deltamm_33" +#endif + }; + description = "1,g1,g2,g3,g4,g5,g5g1,g5g2,g5g3,g5g4,s12,s13,s23,s41,s42,s43"; + + initialize(); + contract_baryons_wall(prop1,prop2,prop3,prop4,*this); +} + + template void PLEGMA_Correlator:: contractBaryonsUDSC(PLEGMA_Propagator &propUP, @@ -246,6 +284,91 @@ contractNucleonThrp_wilsonLine(PLEGMA_Propagator &bwdProp, threep_wilsonLine(*this,bwdProp,fwdProp,signProps,su3,gammas); } +template +void PLEGMA_Correlator:: +contractNucleonThrp_staple(PLEGMA_Propagator &bwdProp, + PLEGMA_Propagator &fwdProp, + PLEGMA_Su3field &su3, + int signProps, std::vector gammas, bool isZfac,int b, int l, int z){ + if(isZfac) shape = {N_SPINS,N_SPINS,N_COLS,N_COLS,(int) gammas.size()}; + else shape = {(int) gammas.size()}; + char d1[50]; + sprintf(d1,"b_%d_l_%d_z_%d",b,l,z); + datasets = {d1}; + groups = {"staple"}; + description = getGammasString(gammas); + initialize(); + + if(gammas.size() == 0) PLEGMA_error("List of gammas provided is empty"); + threep_staple(*this,bwdProp,fwdProp,signProps,su3,gammas,isZfac); +} + +template +void PLEGMA_Correlator:: +contractMesonsFourp_ultralocal(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, PLEGMA_Propagator &prop3, + PLEGMA_Propagator &prop4,int b){ + + shape = {16}; + char d1[50]; + sprintf(d1,"b_%d",b); + datasets = {d1}; + groups = {"mesons"}; + description = "gamma insertion"; + + initialize(); + contract_mesons_fourp_ultralocal(prop1,prop2,prop3,prop4,*this); +} + +template +void PLEGMA_Correlator:: +contractMesonsFourp_ultralocal_oneendtrick(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2,int b){ + + shape = {16}; + char d1[50]; + sprintf(d1,"b_%d",b); + datasets = {d1}; + groups = {"mesons"}; + description = "gamma insertion"; + + initialize(); + contract_mesons_fourp_ultralocal_oneendtrick(prop1,prop2,*this); +} + +template +void PLEGMA_Correlator:: +contractTMDWFMesonsTrick_Zfac(PLEGMA_Propagator &prop1, + PLEGMA_Su3field &staple, int b, int l){ + + shape = {N_SPINS,N_SPINS,N_COLS,N_COLS,16}; + char d1[50]; + sprintf(d1,"b_%d_l_%d",b,l); + datasets = {d1}; + groups = {"mesons"}; + description = "g4 "; + + initialize(); + contract_TMDWF_mesons_trick_zfac(prop1,*this, staple); +} + +template +void PLEGMA_Correlator:: +contractTMDWFMesons_Zfac(PLEGMA_Propagator &prop1, + PLEGMA_Propagator &prop2, + PLEGMA_Su3field &staple, int b, int l){ + + shape = {N_SPINS,N_SPINS,N_COLS,N_COLS,16}; + char d1[50]; + sprintf(d1,"b_%d_l_%d",b,l); + datasets = {d1}; + groups = {"mesons"}; + description = "g4 "; + + initialize(); + contract_TMDWF_mesons_zfac(prop1,prop2,*this, staple); +} + template void PLEGMA_Correlator:: writeASCII(std::string filename_out) const { diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index 17ce2bdc..82ebc7d4 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -1024,6 +1024,11 @@ void PLEGMA_Field::trPmunu(PLEGMA_Gauge &gauge, std::pair trPmunu_k(*this,gauge,munu); } +template +void PLEGMA_Field::SU3Trace(PLEGMA_Su3field &su3field){ + SU3Trace_k(*this,su3field); +} + template class PLEGMA_Field; template class PLEGMA_Field; // Forcing initialization of the following cases diff --git a/lib/PLEGMA_Propagator.cu b/lib/PLEGMA_Propagator.cu index da408ef5..a1e63de0 100644 --- a/lib/PLEGMA_Propagator.cu +++ b/lib/PLEGMA_Propagator.cu @@ -191,6 +191,18 @@ void PLEGMA_Propagator::apply_gamma5(){ apply_gamma5_propagator(*this); } +template +void PLEGMA_Propagator::PropmulVVdag(PLEGMA_Vector &vec1,PLEGMA_Vector &vec2){ + this->zero_device(); + assert(this->checkVolume(vec1)); + assert(this->checkVolume(vec2)); + auto vectex1 = toTexture(vec1); + auto vectex2 = toTexture(vec2); + prop_mul_V_Vdag(toField2(*this), *vectex1, *vectex2); + checkCudaError(); +} + + //----------------------------------// // class PLEGMA_ Propagator3D // //----------------------------------// diff --git a/lib/kernels/PLEGMA_QWF.cuh b/lib/kernels/PLEGMA_QWF.cuh new file mode 100644 index 00000000..0a87ec36 --- /dev/null +++ b/lib/kernels/PLEGMA_QWF.cuh @@ -0,0 +1,346 @@ +#include +using namespace plegma; + +template +__global__ void contract_TMDWF_mesons_trick_zfac_device(propTextexProp1, + su3Tex TexStaple, + int mu, int nu, int c1, int c2, Float2 *block2, int it, int time_step, int maxT, int4 source, bool runFT, tex_mom_list moms){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + register Float2 accum[16]; +#pragma unroll + for(int i = 0 ; i < 16 ; i++){ + accum[i] = 0.; + } + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + texProp1.get(prop1,vid); + Float2 staple[N_COLS][N_COLS]; + TexStaple.get(staple,vid); + + const Float2 (*g)[4]; + const short int (*gIn)[4][2]; + g = (Float2 (*)[4]) plegma::gamma; + gIn = plegma::gammaInd; + +#pragma unroll + for(int ip = 0 ; ip < 16 ; ip++){ +#pragma unroll + for(int g1=0;g1<4;g1++){ + int alpha = gIn[ip][g1][0]; + int beta = gIn[ip][g1][1]; + if (beta == nu){ + Float2 value = g[ip][g1]; +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ + accum[ip] = accum[ip] + value * prop1[mu][alpha][c1][a] * staple[c2][a]; + } + } + } + } + } + + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, 16, sid3D, source_pos, moms, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int ip = 0 ; ip < 16; ip++){ + block2[(tid*DGC_localVolume3D + sid3D)*16 + ip] = accum[ip]; + } + } +} + +template +__global__ void contract_TMDWF_mesons_zfac_device(propTextexProp1, + propTextexProp2, + su3Tex TexStaple, + int mu, int nu, int c1, int c2, Float2 *block2, int it, int time_step, int maxT, int4 source, bool runFT, tex_mom_list moms){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + register Float2 accum[16]; +#pragma unroll + for(int i = 0 ; i < 16 ; i++){ + accum[i] = 0.; + } + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + texProp1.get(prop1,vid); + texProp2.get(prop2,vid); + Float2 staple[N_COLS][N_COLS]; + TexStaple.get(staple,vid); + + const Float2 (*g)[4]; + const short int (*gIn)[4][2]; + g = (Float2 (*)[4]) plegma::gamma; + gIn = plegma::gammaInd; + +#pragma unroll + for(int ip = 0 ; ip < 16 ; ip++){ +#pragma unroll + for(int g1=0;g1<4;g1++){ + int alpha = gIn[ip][g1][0]; + int beta = gIn[ip][g1][1]; + if (beta == nu){ + Float2 value = g[ip][g1]; +#pragma unroll + for(int rho = 0 ; rho < N_SPINS ; rho++){ +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ +#pragma unroll + for(int b = 0 ; b < N_COLS ; b++){ + accum[ip] = accum[ip] + value * prop1[mu][rho][c1][a] * conj(prop2[alpha][rho][b][a]) * staple[c2][b]; + } + } + } + } + } + } + } + + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, 16, sid3D, source_pos, moms, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int ip = 0 ; ip < 16; ip++){ + block2[(tid*DGC_localVolume3D + sid3D)*16 + ip] = accum[ip]; + } + } +} + +template +void contract_TMDWF_mesons_trick_zfac_host( ProfileStruct &ps,PLEGMA_Propagator& prop1, + PLEGMA_Correlator& corr, PLEGMA_Su3field& staple,Float2 *result){ + + int extra = N_SPINS*N_SPINS*N_COLS*N_COLS; + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + size_t size = corr.getTotalSize()/extra/t_size*time_step; + size_t volume = corr.getVolSize()/t_size; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + int site_size = corr.getSiteSize()/extra; + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true) ? (size * (ps.tp.grid.x/time_step)) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto stapleTex = toTexture(staple); + + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess || h_partial_block==NULL) { + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); + return; + } + + for(int it=0; it < t_size; it+=time_step) { + for(int et=0; et < extra; et++){ + int mu=et/N_SPINS/N_COLS/N_COLS; + int nu=(et/N_COLS/N_COLS)%N_SPINS; + int c1=(et/N_COLS)%N_COLS; + int c2=et%N_COLS; + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_TMDWF_mesons_trick_zfac_device + <<>> + (*propTex1, *stapleTex, mu, nu, c1, c2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true) { + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[((it*volume+v)*extra+et)*site_size+f] = 0; + for(int j = 0 ; j < accumX; j++) + result[((it*volume+v)*extra+et)*site_size+f]+= h_partial_block[(v*site_size+f)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[((it*volume+v)*extra+et)*site_size+f] += h_partial_block[v*site_size+f]; + } + } + + } + } + + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); +} + +template +void contract_TMDWF_mesons_zfac_host( ProfileStruct &ps,PLEGMA_Propagator& prop1,PLEGMA_Propagator& prop2, + PLEGMA_Correlator& corr, PLEGMA_Su3field& staple,Float2 *result){ + + int extra = N_SPINS*N_SPINS*N_COLS*N_COLS; + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + size_t size = corr.getTotalSize()/extra/t_size*time_step; + size_t volume = corr.getVolSize()/t_size; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + int site_size = corr.getSiteSize()/extra; + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true) ? (size * (ps.tp.grid.x/time_step)) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + auto stapleTex = toTexture(staple); + + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess || h_partial_block==NULL) { + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); + return; + } + + for(int it=0; it < t_size; it+=time_step) { + for(int et=0; et < extra; et++){ + int mu=et/N_SPINS/N_COLS/N_COLS; + int nu=(et/N_COLS/N_COLS)%N_SPINS; + int c1=(et/N_COLS)%N_COLS; + int c2=et%N_COLS; + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_TMDWF_mesons_zfac_device + <<>> + (*propTex1, *propTex2, *stapleTex, mu, nu, c1, c2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true) { + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[((it*volume+v)*extra+et)*site_size+f] = 0; + for(int j = 0 ; j < accumX; j++) + result[((it*volume+v)*extra+et)*site_size+f]+= h_partial_block[(v*site_size+f)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[((it*volume+v)*extra+et)*site_size+f] += h_partial_block[v*site_size+f]; + } + } + + } + } + + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); +} + +template +static void contract_TMDWF_mesons_trick_zfac(PLEGMA_Propagator& prop1,PLEGMA_Correlator& corr, PLEGMA_Su3field& staple){ + + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size = N_SPINS*N_SPINS*N_COLS*N_COLS*16; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + site_size=16; + + int shared_size = (runFT==true) ? site_size*sizeof(Float2) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + tuneAndRun( ps, "contract_TMDWF_mesons_trick_zfac", contract_TMDWF_mesons_trick_zfac_host, + ps, prop1, corr, staple, result); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(), MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +} + +template +static void contract_TMDWF_mesons_zfac(PLEGMA_Propagator& prop1,PLEGMA_Propagator& prop2,PLEGMA_Correlator& corr, PLEGMA_Su3field& staple){ + + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size = N_SPINS*N_SPINS*N_COLS*N_COLS*16; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + site_size=16; + + int shared_size = (runFT==true) ? site_size*sizeof(Float2) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + tuneAndRun( ps, "contract_TMDWF_mesons_zfac", contract_TMDWF_mesons_zfac_host, + ps, prop1, prop2, corr, staple, result); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(), MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +} diff --git a/lib/kernels/PLEGMA_baryons.cuh b/lib/kernels/PLEGMA_baryons.cuh index 89d04127..8771a2f0 100644 --- a/lib/kernels/PLEGMA_baryons.cuh +++ b/lib/kernels/PLEGMA_baryons.cuh @@ -13,6 +13,9 @@ enum BARYONS_TYPE{NtoN, template __device__ void contract_NtoN_kernel(propTex& texProp1, propTex& texProp2, Float2 accum[2*N_SPINS*N_SPINS], int vid); +template +__device__ void contract_NtoN_wall_kernel(propTex& texProp1, propTex& texProp2, propTex& texProp3, propTex& texProp4, Float2 accum[2*N_SPINS*N_SPINS], int vid); + template __device__ void contract_NtoR_kernel(propTex& texProp1, propTex& texProp2, Float2 accum[2*N_SPINS*N_SPINS], int vid); @@ -95,6 +98,73 @@ __global__ void contract_baryons_device(propTex texProp1, propTex +__global__ void contract_baryons_wall_device(propTex texProp1, propTex texProp2, propTex texProp3, propTex texProp4, Float2* block2, + int it, int time_step, int maxT, int4 source, BARYONS_TYPE ip, bool runFT, tex_mom_list mom_list){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + Float2 accum[2*N_SPINS*N_SPINS]; + + for(int i = 0 ; i < 2*N_SPINS*N_SPINS ; i++){ + accum[i]=0; + } + if (sid3D < DGC_localVolume3D){ // I work only on the spatial volume + switch(ip){ + case NtoN: + contract_NtoN_wall_kernel(texProp1, texProp2, texProp3, texProp4, accum, vid); + break; +#ifdef PLEGMA_LIGHT_BARYONS + case NtoR: + contract_NtoR_kernel(texProp1, texProp2, accum, vid); + break; + case RtoN: + contract_RtoN_kernel(texProp1, texProp2, accum, vid); + break; + case RtoR: + contract_RtoR_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_1O2_1: + contract_deltas_iso1o2_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_1O2_2: + contract_deltas_iso1o2_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_1O2_3: + contract_deltas_iso1o2_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_3O2_1: + contract_deltas_iso3o2_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_3O2_2: + contract_deltas_iso3o2_kernel(texProp1, texProp2, accum, vid); + break; + case DELTA_3O2_3: + contract_deltas_iso3o2_kernel(texProp1, texProp2, accum, vid); + break; +#endif + } + } + + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, 2*N_SPINS*N_SPINS, sid3D, source_pos, mom_list, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int i = 0 ; i < 2*N_SPINS*N_SPINS ; i++){ + block2[(tid*DGC_localVolume3D + sid3D)*2*N_SPINS*N_SPINS + i] = accum[i]; + } + } +} + template static void contract_baryons_host( ProfileStruct &ps, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, @@ -165,6 +235,78 @@ static void contract_baryons_host( ProfileStruct &ps, cudaFree(d_partial_block); } +template +static void contract_baryons_wall_host( ProfileStruct &ps, + PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, PLEGMA_Propagator& prop3, PLEGMA_Propagator& prop4, + PLEGMA_Correlator &corr, Float2 *result, int ip){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = corr.getCorrSpace()==MOMENTUM_SPACE; + size_t volume = corr.getVolSize()/t_size; + size_t size = corr.getTotalSize()/t_size/N_BARYONS*time_step; + int site_size=2*N_SPINS*N_SPINS; + int4 source = corr.getSource(); + auto mom_list = corr.getTexMomList(); + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + // Checking for allocation error. In case we return and let the tuner handle the error. + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess) { + cudaFree(d_partial_block); + return; + } + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + auto propTex3 = toTexture(prop3); + auto propTex4 = toTexture(prop4); + + for(int it=0; it < t_size; it+=time_step) { + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_baryons_wall_device + <<>> + (*propTex1, *propTex2, *propTex3, *propTex4, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, + (BARYONS_TYPE) ip, runFT, *mom_list); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2) , cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true){ + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < 2; f++) + for(int i = 0 ; i < site_size/2; i++) { + result[((f*t_size + it)*volume +v)*site_size/2+i] = 0; + for(int j = 0 ; j < accumX; j++) + result[((f*t_size + it)*volume +v)*site_size/2+i] += + h_partial_block[((v*2+f)*site_size/2+i)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < 2; f++) + for(int i = 0 ; i < site_size/2; i++) + result[((f*t_size + it)*volume +v)*site_size/2+i] = + h_partial_block[(v*2+f)*site_size/2+i]; + + } + } + hostFree(h_partial_block, alloc_size*sizeof(Float2)); + cudaFree(d_partial_block); +} + template static void contract_baryons(PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, PLEGMA_Correlator &corr){ bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); @@ -203,3 +345,43 @@ static void contract_baryons(PLEGMA_Propagator& prop1, PLEGMA_Propagator if(runFT) hostFree(result, (corr.getTotalSize()/N_BARYONS)*sizeof(Float2)); } + +template +static void contract_baryons_wall(PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, PLEGMA_Propagator& prop3, PLEGMA_Propagator& prop4, PLEGMA_Correlator &corr){ + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size=2*N_SPINS*N_SPINS; + + if(corr.getSiteSize()/N_BARYONS != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize()/N_BARYONS, site_size); + + int shared_size = (runFT==true) ? (site_size*sizeof(Float2)) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, (corr.getTotalSize()/N_BARYONS)*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + for(int ip=0; ip, + ps, prop1, prop2, prop3, prop4, corr, result, ip); + if(runFT) { + FloatC *corr_ip = corr.H_elem() + ip*corr.getTotalSize()/N_BARYONS*2; + MPI_Allreduce(result, corr_ip, corr.getTotalSize()/N_BARYONS*2, MPI_Type(corr_ip), + MPI_SUM, HGC_spaceComm); + } else { + result += corr.getTotalSize()/N_BARYONS; + } + } + if(runFT) + hostFree(result, (corr.getTotalSize()/N_BARYONS)*sizeof(Float2)); +} + \ No newline at end of file diff --git a/lib/kernels/PLEGMA_baryons_NtoN.cu b/lib/kernels/PLEGMA_baryons_NtoN.cu index 8f1c2794..b2873a5e 100644 --- a/lib/kernels/PLEGMA_baryons_NtoN.cu +++ b/lib/kernels/PLEGMA_baryons_NtoN.cu @@ -45,7 +45,57 @@ __device__ void contract_NtoN_kernel(propTex& texProp1, propTex& } } +template +__device__ void contract_NtoN_wall_kernel(propTex& texProp1, propTex& texProp2, propTex& texProp3, propTex& texProp4, Float2 accum[2*N_SPINS*N_SPINS], int vid) { + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop3[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop4[N_SPINS][N_SPINS][N_COLS][N_COLS]; + + texProp1.get(prop1,vid); + texProp2.get(prop2,vid); + texProp3.get(prop3,vid); + texProp4.get(prop4,vid); + +#pragma unroll + for(int mu = 0 ; mu < 4 ; mu++) +#pragma unroll + for(int mup = 0 ; mup < 4 ; mup++) { + Float2 tmp[2] = {0., 0.}; +#pragma unroll + for(int idx = 0 ; idx < 16 ; idx++){ + int nu = NtoN_indices[idx][0]; + int lu = NtoN_indices[idx][1]; + int lup = NtoN_indices[idx][2]; + int nup = NtoN_indices[idx][3]; +#pragma unroll + for(int cc1 = 0 ; cc1 < 6 ; cc1++){ + int a = eps[cc1][0]; + int b = eps[cc1][1]; + int c = eps[cc1][2]; +#pragma unroll + for(int cc2 = 0 ; cc2 < 6 ; cc2++){ + int a1 = eps[cc2][0]; + int b1 = eps[cc2][1]; + int c1 = eps[cc2][2]; + FloatC factor = sgn_eps[cc1] * sgn_eps[cc2] * NtoN_values[idx]; + tmp[0] = tmp[0] + factor * prop2[lu][lup][b][b1] * + (prop1[nu][nup][a][a1] * prop1[mu][mup][c][c1] - prop1[nu][mup][a][c1] * prop1[mu][nup][c][a1]); + tmp[1] = tmp[1] + factor * prop4[lu][lup][b][b1] * + (prop3[nu][nup][a][a1] * prop3[mu][mup][c][c1] - prop3[nu][mup][a][c1] * prop3[mu][nup][c][a1]); + } + } + } +#pragma unroll + for(int i = 0 ; i < 2 ; i++){ + accum[(i*N_SPINS + mu)*N_SPINS+mup] = tmp[i]; + } + } +} + template __device__ void contract_NtoN_kernel(propTex& texProp1, propTex& texProp2, Float2 accum[2*N_SPINS*N_SPINS], int vid); template __device__ void contract_NtoN_kernel(propTex& texProp1, propTex& texProp2, Float2 accum[2*N_SPINS*N_SPINS], int vid); +template __device__ void contract_NtoN_wall_kernel(propTex& texProp1, propTex& texProp2, propTex& texProp3, propTex& texProp4, Float2 accum[2*N_SPINS*N_SPINS], int vid); +template __device__ void contract_NtoN_wall_kernel(propTex& texProp1, propTex& texProp2, propTex& texProp3, propTex& texProp4, Float2 accum[2*N_SPINS*N_SPINS], int vid); diff --git a/lib/kernels/PLEGMA_field_utils.cuh b/lib/kernels/PLEGMA_field_utils.cuh index 579d393e..aae6b50e 100644 --- a/lib/kernels/PLEGMA_field_utils.cuh +++ b/lib/kernels/PLEGMA_field_utils.cuh @@ -346,6 +346,16 @@ static void __global__ trPmunu_kernel(FloatA *out, gauge2 u, int mu, int out2[sid]= U3[0][0] + U3[1][1] + U3[2][2]; } +template +static void __global__ SU3Trace_kernel(FloatA *out, su3_2 su3){ + int sid = blockIdx.x*blockDim.x + threadIdx.x; + if (sid >= DGC_localVolume) return; + Float2 *out2 = (Float2 *) out; + Float2 SU3[N_COLS][N_COLS]; + su3.get(SU3,sid); + out2[sid]= SU3[0][0] + SU3[1][1] + SU3[2][2]; +} + template static void trPmunu_k(PLEGMA_Field &f,PLEGMA_Gauge &gauge, std::pair munu){ assert(f.checkVolume(gauge)); @@ -354,3 +364,11 @@ static void trPmunu_k(PLEGMA_Field &f,PLEGMA_Gauge &gauge, std:: tuneAndRun(ps,"trPmunu_kernel",trPmunu_kernel,f.D_elem(),toField2(gauge),std::get<0>(munu),std::get<1>(munu)); checkCudaError(); } + +template +static void SU3Trace_k(PLEGMA_Field &f,PLEGMA_Su3field &su3field){ + assert(f.checkVolume(su3field)); + ProfileStruct ps(su3field.Total_length()); + tuneAndRun(ps,"SU3Trace_kernel",SU3Trace_kernel,f.D_elem(),toField2(su3field)); + checkCudaError(); +} diff --git a/lib/kernels/PLEGMA_mesons.cuh b/lib/kernels/PLEGMA_mesons.cuh index 5a766267..ecb10017 100644 --- a/lib/kernels/PLEGMA_mesons.cuh +++ b/lib/kernels/PLEGMA_mesons.cuh @@ -1,4 +1,5 @@ #include +#pragma once using namespace plegma; const int N_MESONS=10; // TODO: This is hard to extend. These variables should replaced by compile-time functions. @@ -64,6 +65,154 @@ __global__ void contract_mesons_device( propTex texProp1, } } +template +__global__ void contract_mesons_fourp_ultralocal_device( propTex texProp1, + propTex texProp2, + propTex texProp3, + propTex texProp4, + Float2 *block2, + int it, int time_step, int maxT, int4 source, bool runFT, tex_mom_list moms){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + register Float2 accum[16]; + for(int i = 0 ; i < 16 ; i++){ + accum[i] = 0.; + } + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop3[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop4[N_SPINS][N_SPINS][N_COLS][N_COLS]; + texProp1.get(prop1,vid); + texProp2.get(prop2,vid); + texProp3.get(prop3,vid); + texProp4.get(prop4,vid); + + const Float2 (*g)[4]; + const short int (*gIn)[4][2]; + g = (Float2 (*)[4]) plegma::gamma; + gIn = plegma::gammaInd; + +#pragma unroll + for(int ip = 0 ; ip < 16 ; ip++){ +#pragma unroll + for(int alpha = 0 ; alpha < N_SPINS ; alpha++){ +#pragma unroll + for(int delta = 0 ; delta < N_SPINS ; delta++){ +#pragma unroll + for(int g1 = 0 ; g1 < 4 ; g1++){ + int mu = gIn[ip][g1][0]; + int nu = gIn[ip][g1][1]; + Float2 val1 = g[ip][g1]; +#pragma unroll + for(int g2 = 0 ; g2 < 4 ; g2++){ + int rho = gIn[ip][g2][0]; + int sigma = gIn[ip][g2][1]; + Float2 val2 = g[ip][g2]; +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ +#pragma unroll + for(int b = 0 ; b < N_COLS ; b++){ +#pragma unroll + for(int c = 0 ; c < N_COLS ; c++){ +#pragma unroll + for(int d = 0 ; d < N_COLS ; d++){ + accum[ip] += val1 * val2 * conj(prop1[mu][alpha][b][a]) * prop2[nu][delta][b][c] * conj(prop3[rho][delta][d][c]) * prop4[sigma][alpha][d][a]; + } + } + } + } + } + } + } + } + } + } + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, 16, sid3D, source_pos, moms, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int ip = 0 ; ip < 16 ; ip++){ + block2[(tid*DGC_localVolume3D + sid3D)*16 + ip] = accum[ip]; + } + } +} + +template +__global__ void contract_mesons_fourp_ultralocal_oneendtrick_device( propTex texProp1, propTex texProp2, Float2 *block2, int it, int time_step, int maxT, int4 source, bool runFT, tex_mom_list moms){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + register Float2 accum[16]; + for(int i = 0 ; i < 16 ; i++){ + accum[i] = 0.; + } + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + texProp1.get(prop1,vid); + texProp2.get(prop2,vid); + + const Float2 (*g)[4]; + const short int (*gIn)[4][2]; + g = (Float2 (*)[4]) plegma::gamma; + gIn = plegma::gammaInd; + +#pragma unroll + for(int ip = 0 ; ip < 16 ; ip++){ +#pragma unroll + for(int g1 = 0 ; g1 < 4 ; g1++){ + int mu = gIn[ip][g1][0]; + int nu = gIn[ip][g1][1]; + Float2 val1 = g[ip][g1]; +#pragma unroll + for(int g2 = 0 ; g2 < 4 ; g2++){ + int rho = gIn[ip][g2][0]; + int sigma = gIn[ip][g2][1]; + Float2 val2 = g[ip][g2]; +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ +#pragma unroll + for(int b = 0 ; b < N_COLS ; b++){ + accum[ip] += val1 * val2 * prop1[nu][rho][a][b] * prop2[sigma][mu][b][a]; + } + } + } + } + } + } + + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, 16, sid3D, source_pos, moms, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int ip = 0 ; ip < 16 ; ip++){ + block2[(tid*DGC_localVolume3D + sid3D)*16 + ip] = accum[ip]; + } + } +} + template void contract_mesons_host( ProfileStruct &ps, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, @@ -128,6 +277,139 @@ void contract_mesons_host( ProfileStruct &ps, cudaFree(d_partial_block); } +template +void contract_mesons_fourp_ultralocal_host( ProfileStruct &ps, + PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2, PLEGMA_Propagator &prop3, PLEGMA_Propagator &prop4, + PLEGMA_Correlator& corr, Float2 *result){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + size_t size = corr.getTotalSize()/t_size*time_step; + size_t volume = corr.getVolSize()/t_size; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + int site_size = 16; + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true) ? (size * (ps.tp.grid.x/time_step)) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + // Checking for allocation error. In case we return and let the tuner handle the error. + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess) { + cudaFree(d_partial_block); + return; + } + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + auto propTex3 = toTexture(prop3); + auto propTex4 = toTexture(prop4); + + for(int it=0; it < t_size; it+=time_step) { + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_mesons_fourp_ultralocal_device + <<>> + (*propTex1, *propTex2, *propTex3, *propTex4, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true) { + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] = 0; + for(int j = 0 ; j < accumX; j++) + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] += h_partial_block[(v*site_size+f)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume; v++) + for(int f = 0 ; f < site_size; f++) { + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] = h_partial_block[v*site_size+f]; + } + } + } + hostFree(h_partial_block, alloc_size*sizeof(FloatE)); + cudaFree(d_partial_block); +} + +template +void contract_mesons_fourp_ultralocal_oneendtrick_host( ProfileStruct &ps, + PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2, + PLEGMA_Correlator& corr, Float2 *result){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + size_t size = corr.getTotalSize()/t_size*time_step; + size_t volume = corr.getVolSize()/t_size; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + int site_size = 16; + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true) ? (size * (ps.tp.grid.x/time_step)) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + // Checking for allocation error. In case we return and let the tuner handle the error. + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess) { + cudaFree(d_partial_block); + return; + } + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + + for(int it=0; it < t_size; it+=time_step) { + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_mesons_fourp_ultralocal_oneendtrick_device + <<>> + (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true) { + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] = 0; + for(int j = 0 ; j < accumX; j++) + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] += h_partial_block[(v*site_size+f)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume; v++) + for(int f = 0 ; f < site_size; f++) { + result[(((f/16)*t_size+it)*volume+v)*16+f % 16] = h_partial_block[v*site_size+f]; + } + } + } + + hostFree(h_partial_block, alloc_size*sizeof(FloatE)); + cudaFree(d_partial_block); +} + template static void contract_mesons(PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, PLEGMA_Correlator& corr){ @@ -160,3 +442,70 @@ static void contract_mesons(PLEGMA_Propagator& prop1, PLEGMA_Propagator< hostFree(result, corr.getTotalSize()*sizeof(Float2)); } } + +template +static void contract_mesons_fourp_ultralocal(PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2, PLEGMA_Propagator &prop3, PLEGMA_Propagator &prop4, PLEGMA_Correlator& corr){ + + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size = 16; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + int shared_size = (runFT==true) ? site_size*sizeof(Float2) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + tuneAndRun( ps, "contract_mesons_fourp_ultralocal", contract_mesons_fourp_ultralocal_host, + ps, prop1, prop2, prop3, prop4, corr, result); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(), MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +} + +template +static void contract_mesons_fourp_ultralocal_oneendtrick(PLEGMA_Propagator &prop1, PLEGMA_Propagator &prop2, PLEGMA_Correlator& corr){ + + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size = 16; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + int shared_size = (runFT==true) ? site_size*sizeof(Float2) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + tuneAndRun( ps, "contract_mesons_fourp_ultralocal_oneendtrick", contract_mesons_fourp_ultralocal_oneendtrick_host, + ps, prop1, prop2, corr, result); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(), MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +} + \ No newline at end of file diff --git a/lib/kernels/PLEGMA_mesonsNew.cuh b/lib/kernels/PLEGMA_mesonsNew.cuh new file mode 100644 index 00000000..151a21fb --- /dev/null +++ b/lib/kernels/PLEGMA_mesonsNew.cuh @@ -0,0 +1,157 @@ +#include +#include +using namespace plegma; + +template +__global__ void contract_mesons_new_device( propTex texProp1, + propTex texProp2, + Float2 *block2, + int it, int time_step, int maxT, int4 source, + bool runFT, tex_mom_list moms){ + + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + register Float2 accum[N_MESONS]; + for(int i = 0 ; i < N_MESONS ; i++){ + accum[i] = 0.; + } + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + texProp1.get(prop1,vid); + texProp2.get(prop2,vid); +#pragma unroll + for(int ip = 0 ; ip < N_MESONS ; ip++){ +#pragma unroll + for(int is = 0 ; is < N_SPINS*N_SPINS ; is++){ + short int beta = mesons_indices[ip][is][0]; + short int gamma = mesons_indices[ip][is][1]; + short int delta = mesons_indices[ip][is][2]; + short int alpha = mesons_indices[ip][is][3]; + FloatC value = mesons_values[ip][is]; +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ +#pragma unroll + for(int b = 0 ; b < N_COLS ; b++){ + accum[ip] = accum[ip] + value * prop1[alpha][beta][a][b] * conj(prop2[delta][gamma][a][b]); + } + } + } + } + } + if(runFT) { + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + int source_pos[3] = {source.x, source.y, source.z}; + fourier_transform_3D(block2, accum, shared_cache, N_MESONS, sid3D, source_pos, moms, 0, -1, time_step, tid); + } else { + if (sid3D < DGC_localVolume3D) + for(int ip = 0 ; ip < N_MESONS ; ip++){ + block2[(tid*DGC_localVolume3D + sid3D)*N_MESONS + ip] = accum[ip]; + } + } +} + +template +void contract_mesons_new_host( ProfileStruct &ps, + PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, + PLEGMA_Correlator& corr, Float2 *result){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + size_t size = corr.getTotalSize()/t_size*time_step; + size_t volume = corr.getVolSize()/t_size; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + int site_size = N_MESONS; + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("t_size = %d, maxT = %d, source.w = %d, time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", t_size, maxT, source.w, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true) ? (size * (ps.tp.grid.x/time_step)) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + // Checking for allocation error. In case we return and let the tuner handle the error. + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess) { + cudaFree(d_partial_block); + return; + } + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + for(int it=0; it < t_size; it+=time_step) { + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*std::min(t_size-it, time_step); + contract_mesons_new_device + <<>> + (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + + if(runFT==true) { + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*std::min(t_size-it, time_step); v++) + for(int f = 0 ; f < site_size; f++) { + result[(it*volume+v)*site_size+f] = 0; + for(int j = 0 ; j < accumX; j++) + result[(it*volume+v)*site_size+f] += h_partial_block[(v*site_size+f)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume; v++) + for(int f = 0 ; f < site_size; f++) { + result[(it*volume+v)*site_size+f] = h_partial_block[v*site_size+f]; + } + } + } + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); +} + +template +static void contract_mesons_new(PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, + PLEGMA_Correlator& corr){ + bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); + int site_size = N_MESONS; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + int shared_size = (runFT==true) ? site_size*sizeof(Float2) : 0; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + ProfileStruct ps(HGC_localVolume3D, shared_size); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + tuneAndRun( ps, "contract_mesons_new", contract_mesons_new_host, + ps, prop1, prop2, corr, result); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(), MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +} \ No newline at end of file diff --git a/lib/kernels/PLEGMA_propagator_utils.cuh b/lib/kernels/PLEGMA_propagator_utils.cuh index 9e2f6611..0c5ad719 100644 --- a/lib/kernels/PLEGMA_propagator_utils.cuh +++ b/lib/kernels/PLEGMA_propagator_utils.cuh @@ -86,7 +86,6 @@ void apply_boundaries(Float *inOut, int t0){ checkCudaError(); } - template static __global__ void rotateToPhysicalBase_kernel(Float *inOut, int sign){ @@ -132,3 +131,43 @@ void rotateToPhysicalBase(Float* inOut, int sign){ rotateToPhysicalBase_kernel<<>>((Float*) inOut,sign); checkCudaError(); } + +template +static __global__ void prop_mul_V_Vdag_kernel(prop2 prop, vectorTex vectex1, vectorTex vectex2) { + + int sid = blockIdx.x*blockDim.x + threadIdx.x; + if (sid >= prop.volume()) return; + + Float2 vc1[N_SPINS][N_COLS]; + Float2 vc2[N_SPINS][N_COLS]; + Float2 prp[N_SPINS][N_SPINS][N_COLS][N_COLS]; + + vectex1.get(vc1,sid); + vectex2.get(vc2,sid); + +#pragma unroll + for(int mu = 0 ; mu < N_SPINS ; mu++){ +#pragma unroll + for(int nu = 0 ; nu < N_SPINS ; nu++){ +#pragma unroll + for(int a = 0 ; a < N_COLS ; a++){ +#pragma unroll + for(int b = 0 ; b < N_COLS ; b++){ + prp[mu][nu][a][b] = vc1[mu][a] * conj(vc2[nu][b]); + } + } + } + } + + prop.set(prp,sid); + +} + + +template +static void prop_mul_V_Vdag(prop2 prop, vectorTex& vectex1, vectorTex& vectex2){ + ProfileStruct ps(prop.volume()); + run(ps, "prop_mul_V_Vdag_kernel", prop_mul_V_Vdag_kernel, prop, vectex1, vectex2); + checkCudaError(); +} + \ No newline at end of file diff --git a/lib/kernels/PLEGMA_threep.cuh b/lib/kernels/PLEGMA_threep.cuh index 17868c84..d5e223c0 100644 --- a/lib/kernels/PLEGMA_threep.cuh +++ b/lib/kernels/PLEGMA_threep.cuh @@ -30,9 +30,9 @@ void threep_threeD_part3(PLEGMA_Correlator &corr, PLEGMA_Propagator void threep_threeD_part4(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac); - - template void threep_wilsonLine(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& gauge, std::vector& gammas); +template +void threep_staple(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& gauge, std::vector& gammas, bool isZfac); diff --git a/lib/kernels/PLEGMA_threep_staple.cu b/lib/kernels/PLEGMA_threep_staple.cu new file mode 100644 index 00000000..9e38039c --- /dev/null +++ b/lib/kernels/PLEGMA_threep_staple.cu @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include +#include + +using namespace plegma; +template +struct KernelArr {T* array; int size;}; + +template +__global__ void threep_staple_device(Float2* block2, + propTex prop1Tex, propTex prop2Tex, + su3Tex su3Tx, KernelArr listGammas, + int it, int time_step, int maxT, int4 source, + int signProps, bool runFT, tex_mom_list moms, int mu, int nu, int c1, int c2){ + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + Float2 accum[N_SPINS*N_SPINS]; // max value of gammas + #pragma unroll + for(int i = 0; i < N_SPINS*N_SPINS; i++) + accum[i]=0; + + if (sid3D < DGC_localVolume3D){ + Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 su3[N_COLS][N_COLS]; + Float2 R[N_SPINS][N_SPINS]; + prop1Tex.get(prop1,vid); + prop2Tex.get(prop2,vid); + su3Tx.get(su3,vid); + + bool notZfac=(mu<0) && (nu<0) && (c1<0) && (c2<0); + if(notZfac) partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3); + else open_mul_Prop_G_Prop(R,prop1,prop2,su3,mu,nu,c1,c2); + + for(int iop = 0; iop < listGammas.size; iop++){ + int opId=listGammas.array[iop]; + if(notZfac){ + accum[iop]=((signProps > 0) ? trace_gamma_S(opId,TMP,R) : trace_gamma_S(opId,TMM,R));} + else{ + accum[iop]=trace_gamma_S(opId,NOROT,R); + } + } + } + + int source_pos[3] = {source.x, source.y, source.z}; + + if(runFT){ + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + fourier_transform_3D(block2, accum, shared_cache, listGammas.size, sid3D, source_pos, moms, 0, +1, time_step, tid); + } else{ + if (sid3D < DGC_localVolume3D) + for(int iop = 0; iop < listGammas.size; iop++) + block2[(tid*DGC_localVolume3D + sid3D)*listGammas.size +iop] = accum[iop]; + } +} + +template +static void threep_staple_host(ProfileStruct &ps, Float2 *result, + PLEGMA_Correlator &corr, + PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, + int signProps, PLEGMA_Su3field& su3, std::vector& gammas, bool isZfac){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + size_t volume = corr.getVolSize()/t_size; + int extra=1; + if(isZfac) extra=N_SPINS*N_SPINS*N_COLS*N_COLS; + size_t size = corr.getTotalSize()/extra/t_size*time_step; + int site_size = corr.getSiteSize()/extra; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + + KernelArr listGammas; + listGammas.size = gammas.size(); + cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); + cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("time_step = %d, t_size = %d, maxT = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", time_step, t_size, maxT, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); + auto propTex2 = toTexture(prop2); + auto su3tex = toTexture(su3); + + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess || h_partial_block==NULL) goto exit; + for(int it=0; it < t_size; it+=time_step) { + for(int et=0; et < extra; et++) { + int mu=-1, nu=-1, c1=-1, c2=-1; + if(isZfac) { + mu=et/N_SPINS/N_COLS/N_COLS; + nu=(et/N_COLS/N_COLS)%N_SPINS; + c1=(et/N_COLS)%N_COLS; + c2=et%N_COLS; + } + int t_step = std::min(t_size-it, time_step); + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*t_step; + threep_staple_device + <<>> + (d_partial_block, *propTex1, *propTex2, *su3tex, listGammas, it, t_step, maxT, + source, signProps, runFT, *moms, mu,nu,c1,c2); + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + if(runFT==true){ + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) { + result[((it*volume+v)*extra+et)*site_size+i] = 0; + for(int j = 0 ; j < accumX; j++) + result[((it*volume+v)*extra+et)*site_size+i] += + h_partial_block[(v*site_size+i)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) + result[((it*volume+v)*extra+et)*site_size+i] += + h_partial_block[v*site_size+i]; + } + } + } + + exit: + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); + cudaFree(listGammas.array); +} + +template +static void threep_staple(PLEGMA_Correlator &corr, + PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, + int signProps, PLEGMA_Su3field& su3, std::vector& gammas,bool isZfac){ +#ifdef PLEGMA_NUCLEON_3PF_FIX_SINK + if(gammas.size() <= 0) + PLEGMA_error("Error the container of gamma matrices cannot be zero"); + if(gammas.size() > N_SPINS*N_SPINS) + PLEGMA_error("Error maximum number of gamma matrices is 16"); + + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + int site_size = gammas.size(); + + if(isZfac) + site_size *= N_SPINS*N_SPINS*N_COLS*N_COLS; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + site_size = gammas.size(); + ProfileStruct ps(HGC_localVolume3D, (runFT==true) ? site_size*sizeof(Float2) : 0); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + tune( ps, "threep_staple", threep_staple_host, + ps, result, corr, prop1, prop2, signProps, su3, gammas, false); + run( ps, "threep_staple", threep_staple_host, + ps, result, corr, prop1, prop2, signProps, su3, gammas, isZfac); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(corr.H_elem()), + MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +#else + PLEGMA_error("You must enable PLEGMA_NUCLEON_3PF_FIX_SINK\n"); +#endif +} + +template void threep_staple(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& su3, std::vector& gammas,bool isZfac); +template void threep_staple(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& su3, std::vector& gammas,bool isZfac); diff --git a/plegma/CMakeLists.txt b/plegma/CMakeLists.txt index ef808191..6ea0c007 100644 --- a/plegma/CMakeLists.txt +++ b/plegma/CMakeLists.txt @@ -46,6 +46,9 @@ target_link_libraries(Calc_2pt ${PLEGMA_LIBS}) cuda_add_executable(PDFs_nucleon PDFs_nucleon.cpp) target_link_libraries(PDFs_nucleon ${PLEGMA_LIBS}) +cuda_add_executable(TMDPDFs_nucleon TMDPDFs_nucleon.cpp) +target_link_libraries(TMDPDFs_nucleon ${PLEGMA_LIBS}) + cuda_add_executable(nucleon_2pt_3pt nucleon_2pt_3pt.cpp) target_link_libraries(nucleon_2pt_3pt ${PLEGMA_LIBS}) @@ -97,6 +100,21 @@ target_link_libraries(testShifts ${PLEGMA_LIBS}) cuda_add_executable(Zfac Zfac.cpp) target_link_libraries(Zfac ${PLEGMA_LIBS}) +cuda_add_executable(ZfacStaple ZfacStaple.cpp) +target_link_libraries(ZfacStaple ${PLEGMA_LIBS}) + +cuda_add_executable(SoftFunction SoftFunction.cpp) +target_link_libraries(SoftFunction ${PLEGMA_LIBS}) + +cuda_add_executable(SoftFunctionWall SoftFunctionWall.cpp) +target_link_libraries(SoftFunctionWall ${PLEGMA_LIBS}) + +cuda_add_executable(WallSourceNucleon WallSourceNucleon.cpp) +target_link_libraries(WallSourceNucleon ${PLEGMA_LIBS}) + +cuda_add_executable(TMDPDFs_pion TMDPDFs_pion.cpp) +target_link_libraries(TMDPDFs_pion ${PLEGMA_LIBS}) + if (PLEGMA_SCATTERING_CONTRACTIONS) cuda_add_executable(testScattReductions EXCLUDE_FROM_ALL testScattReductions.cpp) target_link_libraries(testScattReductions ${PLEGMA_LIBS}) @@ -105,7 +123,7 @@ if (PLEGMA_SCATTERING_CONTRACTIONS) endif(PLEGMA_SCATTERING_CONTRACTIONS) -cuda_add_executable(gaugeFixing EXCLUDE_FROM_ALL gaugeFixing.cpp) +cuda_add_executable(gaugeFixing gaugeFixing.cpp) target_link_libraries(gaugeFixing ${PLEGMA_LIBS}) cuda_add_executable(invert EXCLUDE_FROM_ALL invert.cpp) diff --git a/plegma/SoftFunction.cpp b/plegma/SoftFunction.cpp new file mode 100644 index 00000000..487db44c --- /dev/null +++ b/plegma/SoftFunction.cpp @@ -0,0 +1,461 @@ +#include +#include + +using namespace plegma; +using namespace quda; + +int main(int argc, char **argv) +{ + + static std::vector listOpt = {"verbosity", "load-gauge", "nsmear-APE", "alpha-APE", "nsmear-gauss", "alpha-gauss", + "nsrc", "src-filename", "maxQsq", "twop-filename","threep-filename", "corr-file-format", + "corr-space", "tSinks","Projs","xiMomSm","sinkMom","which_particle","gammas"}; + + initializeOptions(argc, argv, true, listOpt); + + double mu_ud; + + double rhoStout; + HGC_options->set("rho_stout", "Rho parameter stout smearing", verbosity, rhoStout); + + int nStout; + HGC_options->set("n_stout", "n parameter stout smearing", verbosity, nStout); + + int L_LEN; + HGC_options->set("l-len", "length of l", verbosity, L_LEN); + + int L_DIR; + HGC_options->set("l-dir", "direction of l", verbosity, L_DIR); + + int B_LEN; + HGC_options->set("b-len", "length of b", verbosity, B_LEN); + + int B_DIR; + HGC_options->set("b-dir", "direction of b", verbosity, B_DIR); + + int z = 0; + HGC_options->set("z-len", "length of z", verbosity, z); + + int seed=0; + HGC_options->set("seed", "seed", verbosity, seed); + + std::vector PMom = {0,0,0}; + HGC_options->set("P-momentum", "If added to the momentum transfer delta gives the sink momentum", verbosity, PMom); + + std::vector pMom = {0,0,0}; + HGC_options->set("p-momentum", "If added to the momentum transfer delta gives the sink momentum", verbosity, pMom); + + int tsrc; + HGC_options->set("t-source", "Time of source vector", verbosity, tsrc); + + int tsnk; + HGC_options->set("t-sink", "Time of sink vector", verbosity, tsnk); + + std::string fourp_filename; + HGC_options->set("fourp-filename", "Filename of the four-points correlator", verbosity, fourp_filename); + + std::string qwf_filename; + HGC_options->set("qwf-filename", "Filename of the quasi-wave function", verbosity, qwf_filename); + + initializePLEGMA(); + + std::vector PpMom = {0,0,0}; + + std::transform(PMom.begin(), PMom.end(), pMom.begin(), PpMom.begin(), std::minus()); + + std::vector P2Mom = {0,0,0}; + + std::transform(PMom.begin(), PMom.end(), PMom.begin(), P2Mom.begin(), std::plus()); + + std::vector zero_mom = {0,0,0}; + + PLEGMA_Gauge gauge; + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + + initGaugeQuda(gauge, true, QUDA_WILSON_LINKS); + plaqQuda(); + + PLEGMA_Gauge smearedGauge; + PLEGMA_Gauge smearedGaugeMn; + smearedGauge.APEsmearing(gauge, nsmearAPE, alphaAPE, 3); + PLEGMA_printf("Plaquette after smearing:\n"); + smearedGauge.calculatePlaq(); + smearedGaugeMn.copy(smearedGauge); + + std::complex momSmPlScale[N_DIMS]; + std::complex momSmMnScale[N_DIMS]; + std::complex I(0,1); + for(int i = 0 ; i < N_DIMS; i++){ + momSmPlScale[i] = std::exp(-(xiMomSm*2.*PI*sinkMom[i]/HGC_totalL[i])*I); + momSmMnScale[i] = std::exp(+(xiMomSm*2.*PI*sinkMom[i]/HGC_totalL[i])*I); + } + smearedGauge.scaleDirWise(momSmPlScale); + smearedGaugeMn.scaleDirWise(momSmMnScale); + + PLEGMA_Gauge gaugeWL; + gaugeWL.copy(gauge); + gaugeWL.stoutSmearing(gaugeWL,nStout,rhoStout,3); + PLEGMA_Su3field *WL = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field tmp(BOTH); + PLEGMA_Su3field *u_s[4]; + for(int idir = 0; idir < 4 ; idir++){ + u_s[idir] = new PLEGMA_Su3field(BOTH); + u_s[idir]->absorbDir_device(gaugeWL,idir); + } + PLEGMA_Su3field *WLIn = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field *WLExchange = nullptr; + + if(mu<0) mu*=-1.; + mu_ud = mu; + QUDA_solver *solver = new QUDA_solver(mu); + + PLEGMA_Vector vectorIn1; + PLEGMA_Vector vectorIn2; + PLEGMA_Vector vectorDil1[12]; + PLEGMA_Vector vectorDil2[12]; + PLEGMA_Vector vectorC21[12]; + PLEGMA_Vector vectorC22[12]; + PLEGMA_Vector vectorOutUP1[12]; + PLEGMA_Vector vectorOutUP2[12]; + PLEGMA_Vector vectorOutDN1[12]; + PLEGMA_Vector vectorOutDN2[12]; + PLEGMA_Vector vectorAuxD1; + PLEGMA_Vector vectorAuxD2; + PLEGMA_Vector vectorAuxF; + PLEGMA_Propagator tmpprop(BOTH); + PLEGMA_Propagator prop1(BOTH); + PLEGMA_Propagator prop2(BOTH); + + vectorIn1.randInit(1234+seed); + vectorIn2.randInit(5678+seed); + vectorIn1.stochastic_Z(4); + vectorIn2.stochastic_Z(4); + for(int i=0;i<12;i++){ + vectorDil1[i].dilutespincolor(vectorIn1,i%4,i/4); + vectorDil2[i].dilutespincolor(vectorIn2,i%4,i/4); + } + + site& source = sourcePositions[0]; + + if(mu != mu_ud) { + mu = mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD1.absorbTimeslice(vectorDil1[isc],tsrc,true); + vectorAuxD1.mulMomentumPhases((std::vector) {pMom[0],pMom[1],pMom[2],0},-1); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGaugeMn, nsmearGauss, alphaGauss); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,+1); + solver->solve(vectorAuxD2,vectorAuxD1); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,+1); + vectorOutUP1[isc].copy(vectorAuxD1); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD1.absorbTimeslice(vectorDil1[isc],tsrc,true); + vectorAuxD1.mulMomentumPhases((std::vector) {PpMom[0],PpMom[1],PpMom[2],0},+1); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGauge, nsmearGauss, alphaGauss); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,+1); + solver->solve(vectorAuxD2,vectorAuxD1); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,+1); + vectorOutUP2[isc].copy(vectorAuxD1); + } + + if(mu != -mu_ud) { + mu = -mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD1.absorbTimeslice(vectorDil2[isc],tsnk,true); + vectorAuxD1.mulMomentumPhases((std::vector) {pMom[0],pMom[1],pMom[2],0},+1); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGauge, nsmearGauss, alphaGauss); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,-1); + solver->solve(vectorAuxD2,vectorAuxD1); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,-1); + vectorOutDN1[isc].copy(vectorAuxD1); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD1.absorbTimeslice(vectorDil2[isc],tsnk,true); + vectorAuxD1.mulMomentumPhases((std::vector) {PpMom[0],PpMom[1],PpMom[2],0},-1); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGaugeMn, nsmearGauss, alphaGauss); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,-1); + solver->solve(vectorAuxD2,vectorAuxD1); + vectorAuxD1.rotateToPhysicalBasis(vectorAuxD2,-1); + vectorOutDN2[isc].copy(vectorAuxD1); + } + + for(int i=0;i<12;i++){ + vectorAuxD1.copy(vectorOutUP2[i]); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGauge, nsmearGauss, alphaGauss); + vectorC21[i].copy(vectorAuxD2); + vectorAuxD1.copy(vectorOutUP1[i]); + vectorAuxD2.gaussianSmearing(vectorAuxD1, smearedGaugeMn, nsmearGauss, alphaGauss); + vectorC22[i].copy(vectorAuxD2); + } + + prop1.zero_where(BOTH); + for(int j=0 ; j<12 ; j++){ + tmpprop.PropmulVVdag(vectorC21[j],vectorC22[j]); + prop1.add(tmpprop,1.0); + } + + WL->setUnit((std::vector) {0,4,8}); + + PLEGMA_Correlator corrqwf(corr_space, source); + corrqwf.setFixMomVec(sinkMom); + corrqwf.contractTMDWFMesonsTrick_Zfac(prop1,*WL,0,0); + corrqwf.writeFile(twop_filename.c_str(), corr_file_format); + + PLEGMA_Correlator corr4p(corr_space, source); + corr4p.setFixMomVec(P2Mom); + + for(int b=0;b0){ + for(int isc=0;isc<12;isc++){ + vectorAuxF.shift(vectorOutUP1[isc], 4+B_DIR); + vectorOutUP1[isc].copy(vectorAuxF); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxF.shift(vectorOutDN1[isc], 4+B_DIR); + vectorOutDN1[isc].copy(vectorAuxF); + } + } + + prop1.zero_where(BOTH); + prop2.zero_where(BOTH); + + for(int i=0 ; i<12 ; i++){ + tmpprop.PropmulVVdag(vectorOutDN1[i],vectorOutDN2[i]); + prop1.add(tmpprop,1.0); + } + + for(int j=0 ; j<12 ; j++){ + tmpprop.PropmulVVdag(vectorOutUP2[j],vectorOutUP1[j]); + prop2.add(tmpprop,1.0); + } + + corr4p.contractMesonsFourp_ultralocal_oneendtrick(prop1, prop2,b); + corr4p.writeFile(fourp_filename.c_str(), corr_file_format); + + for(int l=0;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z]; + + if((l!=0)||(b!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+B_DIR); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + + } + + WL->conjugate(); + corrqwf.contractTMDWFMesonsTrick_Zfac(prop2,*WL,b,l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + + for(int l=1;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b-z]; + + for(int i=0;i vspath(spath,spath+(2*l)+b-z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b-z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+B_DIR); + } + for(int i=0;ishift(*WLIn,4+L_DIR); + } + + WL->conjugate(); + corrqwf.contractTMDWFMesonsTrick_Zfac(prop2,*WL,b,-l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + } + + for(int b=0;b0){ + for(int isc=0;isc<12;isc++){ + vectorAuxF.shift(vectorOutUP1[isc], B_DIR); + vectorOutUP1[isc].copy(vectorAuxF); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxF.shift(vectorOutDN1[isc], B_DIR); + vectorOutDN1[isc].copy(vectorAuxF); + } + } + + prop1.zero_where(BOTH); + prop2.zero_where(BOTH); + + for(int i=0 ; i<12 ; i++){ + tmpprop.PropmulVVdag(vectorOutDN1[i],vectorOutDN2[i]); + prop1.add(tmpprop,1.0); + } + + for(int j=0 ; j<12 ; j++){ + tmpprop.PropmulVVdag(vectorOutUP2[j],vectorOutUP1[j]); + prop2.add(tmpprop,1.0); + } + + corr4p.contractMesonsFourp_ultralocal_oneendtrick(prop1, prop2,-b); + corr4p.writeFile(fourp_filename.c_str(), corr_file_format); + + for(int l=0;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z]; + + if((l!=0)||(b!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,B_DIR); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + } + + WL->conjugate(); + corrqwf.contractTMDWFMesonsTrick_Zfac(prop2,*WL,-b,l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + + for(int l=1;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b-z]; + + for(int i=0;i vspath(spath,spath+(2*l)+b-z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b-z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,B_DIR); + } + for(int i=0;ishift(*WLIn,4+L_DIR); + } + + WL->conjugate(); + corrqwf.contractTMDWFMesonsTrick_Zfac(prop2,*WL,-b,-l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + } + + for(int idir = 0; idir < 4 ; idir++){ + delete u_s[idir]; + } + delete WL; + delete WLIn; + + delete solver; + + finalize(); + + return 0; +} diff --git a/plegma/SoftFunctionWall.cpp b/plegma/SoftFunctionWall.cpp new file mode 100644 index 00000000..702c776c --- /dev/null +++ b/plegma/SoftFunctionWall.cpp @@ -0,0 +1,406 @@ +#include +#include + +using namespace plegma; +using namespace quda; + +int main(int argc, char **argv) +{ + + static std::vector listOpt = {"verbosity", "load-gauge", "nsmear-APE", "alpha-APE", "nsmear-gauss", "alpha-gauss", + "nsrc", "src-filename", "maxQsq", "twop-filename","threep-filename", "corr-file-format", + "corr-space", "tSinks","Projs","xiMomSm","sinkMom","which_particle","gammas"}; + + initializeOptions(argc, argv, true, listOpt); + + double mu_ud; + + double rhoStout; + HGC_options->set("rho_stout", "Rho parameter stout smearing", verbosity, rhoStout); + + int nStout; + HGC_options->set("n_stout", "n parameter stout smearing", verbosity, nStout); + + int L_LEN; + HGC_options->set("l-len", "length of l", verbosity, L_LEN); + + int L_DIR; + HGC_options->set("l-dir", "direction of l", verbosity, L_DIR); + + int B_LEN; + HGC_options->set("b-len", "length of b", verbosity, B_LEN); + + int B_DIR; + HGC_options->set("b-dir", "direction of b", verbosity, B_DIR); + + int z = 0; + HGC_options->set("z-len", "length of z", verbosity, z); + + std::vector PMom = {0,0,0}; + HGC_options->set("P-momentum", "If added to the momentum transfer delta gives the sink momentum", verbosity, PMom); + + std::vector pMom = {0,0,0}; + HGC_options->set("p-momentum", "If added to the momentum transfer delta gives the sink momentum", verbosity, pMom); + + int tsrc; + HGC_options->set("t-source", "Time of source vector", verbosity, tsrc); + + int tsnk; + HGC_options->set("t-sink", "Time of sink vector", verbosity, tsnk); + + std::string fourp_filename; + HGC_options->set("fourp-filename", "Filename of the four-points correlator", verbosity, fourp_filename); + + std::string qwf_filename; + HGC_options->set("qwf-filename", "Filename of the quasi-wave function", verbosity, qwf_filename); + + std::string ft1_filename; + std::string ft2_filename; + HGC_options->set("ft1-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft1_filename); + HGC_options->set("ft2-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft2_filename); + + initializePLEGMA(); + + std::vector PpMom = {0,0,0}; + + std::transform(PMom.begin(), PMom.end(), pMom.begin(), PpMom.begin(), std::minus()); + + std::vector P2Mom = {0,0,0}; + + std::transform(PMom.begin(), PMom.end(), PMom.begin(), P2Mom.begin(), std::plus()); + + std::vector zero_mom = {0,0,0}; + + PLEGMA_Gauge gauge; + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + + initGaugeQuda(gauge, true, QUDA_WILSON_LINKS); + plaqQuda(); + + PLEGMA_Gauge smearedGauge; + smearedGauge.APEsmearing(gauge, nsmearAPE, alphaAPE, 3); + PLEGMA_printf("Plaquette after smearing:\n"); + smearedGauge.calculatePlaq(); + + PLEGMA_Gauge gaugeWL; + gaugeWL.copy(gauge); + gaugeWL.stoutSmearing(gaugeWL,nStout,rhoStout,3); + PLEGMA_Su3field *WL = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field tmp(BOTH); + PLEGMA_Su3field *u_s[4]; + for(int idir = 0; idir < 4 ; idir++){ + u_s[idir] = new PLEGMA_Su3field(BOTH); + u_s[idir]->absorbDir_device(gaugeWL,idir); + } + PLEGMA_Su3field *WLIn = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field *WLExchange = nullptr; + + if(mu<0) mu*=-1.; + mu_ud = mu; + QUDA_solver *solver = new QUDA_solver(mu); + + PLEGMA_Vector vectorIn; + PLEGMA_Vector vectorOut; + PLEGMA_Vector vectorAuxD; + PLEGMA_Vector vectorAuxF; + + PLEGMA_Propagator *propUPp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propDNp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propUPPp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propDNPp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propIn = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propExchange = nullptr; + + site& source = sourcePositions[0]; + + if(mu != mu_ud) { + mu = mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + vectorIn.mulMomentumPhases((std::vector) {pMom[0],pMom[1],pMom[2],0},-1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propUPp->absorb(vectorAuxF, isc/3, isc%3); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + vectorIn.mulMomentumPhases((std::vector) {PpMom[0],PpMom[1],PpMom[2],0},+1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propUPPp->absorb(vectorAuxF, isc/3, isc%3); + } + + propUPPp->rotateToPhysicalBase_device(+1); + propUPp->rotateToPhysicalBase_device(+1); + propUPPp->applyBoundaries_device(0); + propUPp->applyBoundaries_device(0); + + PLEGMA_FT ftp(pMom,3); + PLEGMA_FT ftPp(PpMom,3); + ftp.apply(*propUPp,FT_GEMV,+1); + ftPp.apply(*propUPPp); + ftp.writeFile(ft1_filename, corr_file_format); + ftPp.writeFile(ft2_filename, corr_file_format); + + WL->setUnit((std::vector) {0,4,8}); + + PLEGMA_Correlator corr(corr_space, source); + corr.setFixMomVec(sinkMom); + corr.contractTMDWFMesons_Zfac(*propUPPp,*propUPp,*WL,0,0); + corr.writeFile(twop_filename, corr_file_format); + + if(mu != -mu_ud) { + mu = -mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsnk,true); + vectorIn.mulMomentumPhases((std::vector) {pMom[0],pMom[1],pMom[2],0},+1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propDNp->absorb(vectorAuxF, isc/3, isc%3); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsnk,true); + vectorIn.mulMomentumPhases((std::vector) {PpMom[0],PpMom[1],PpMom[2],0},-1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propDNPp->absorb(vectorAuxF, isc/3, isc%3); + } + + propDNp->rotateToPhysicalBase_device(-1); + propDNp->applyBoundaries_device(source[0]); + propDNPp->rotateToPhysicalBase_device(-1); + propDNPp->applyBoundaries_device(source[0]); + + PLEGMA_Correlator corr4p(corr_space, source); + corr4p.setFixMomVec(P2Mom); + + PLEGMA_Correlator corrqwf(corr_space, source); + corrqwf.setFixMomVec(sinkMom); + + for(int b=0;b0){ + propExchange = propIn; propIn = propUPp; propUPp = propExchange; + propUPp->shift(*propIn, 4+B_DIR); + + propExchange = propIn; propIn = propDNp; propDNp = propExchange; + propDNp->shift(*propIn, 4+B_DIR); + } + + corr4p.contractMesonsFourp_ultralocal(*propUPp, *propDNp, *propDNPp, *propUPPp,b); + corr4p.writeFile(fourp_filename.c_str(), corr_file_format); + + for(int l=0;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z]; + + if((l!=0)||(b!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+B_DIR); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + + } + + WL->conjugate(); + corrqwf.contractTMDWFMesons_Zfac(*propUPPp,*propUPp,*WL,b,l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + } + + for(int l=1;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b-z]; + + for(int i=0;i vspath(spath,spath+(2*l)+b-z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b-z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+B_DIR); + } + for(int i=0;ishift(*WLIn,4+L_DIR); + } + + WL->conjugate(); + corrqwf.contractTMDWFMesons_Zfac(*propUPPp,*propUPp,*WL,b,-l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + } + + for(int b=0;bshift(*propIn, B_DIR); + + propExchange = propIn; propIn = propDNp; propDNp = propExchange; + propDNp->shift(*propIn, B_DIR); + } + + for(int b=1;b0){ + propExchange = propIn; propIn = propUPp; propUPp = propExchange; + propUPp->shift(*propIn, B_DIR); + + propExchange = propIn; propIn = propDNp; propDNp = propExchange; + propDNp->shift(*propIn, B_DIR); + } + + corr4p.contractMesonsFourp_ultralocal(*propUPp, *propDNp, *propDNPp, *propUPPp,-b); + corr4p.writeFile(fourp_filename.c_str(), corr_file_format); + + for(int l=0;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z]; + + if((l!=0)||(b!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,4+L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,B_DIR); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + } + + WL->conjugate(); + corrqwf.contractTMDWFMesons_Zfac(*propUPPp,*propUPp,*WL,-b,l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + + for(int l=1;lsetUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b-z]; + + for(int i=0;i vspath(spath,spath+(2*l)+b-z); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b-z);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,B_DIR); + } + for(int i=0;ishift(*WLIn,4+L_DIR); + } + + WL->conjugate(); + corrqwf.contractTMDWFMesons_Zfac(*propUPPp,*propUPp,*WL,-b,-l); + corrqwf.writeFile(qwf_filename.c_str(), corr_file_format); + + } + } + + for(int idir = 0; idir < 4 ; idir++){ + delete u_s[idir]; + } + delete WL; + delete WLIn; + + delete propUPp; + delete propUPPp; + delete propDNp; + delete propDNPp; + delete propIn; + + delete solver; + + finalize(); + + return 0; +} diff --git a/plegma/TMDPDFs_nucleon.cpp b/plegma/TMDPDFs_nucleon.cpp new file mode 100644 index 00000000..463873a2 --- /dev/null +++ b/plegma/TMDPDFs_nucleon.cpp @@ -0,0 +1,630 @@ +#include +#include + +using namespace plegma; +using namespace quda; + +int main(int argc, char **argv) +{ + + static std::vector listOpt = {"verbosity", "load-gauge", "nsmear-APE", "alpha-APE", "nsmear-gauss", "alpha-gauss", + "nsrc", "src-filename", "maxQsq", "twop-filename","threep-filename", "corr-file-format", + "corr-space", "tSinks","Projs","xiMomSm","sinkMom","which_particle","gammas"}; + + initializeOptions(argc, argv, true, listOpt); + + int L_LEN; + HGC_options->set("l-len", "length of l", verbosity, L_LEN); + + int L_DIR; + HGC_options->set("l-dir", "direction of l", verbosity, L_DIR); + + int B_LEN; + HGC_options->set("b-len", "length of b", verbosity, B_LEN); + + int B_DIR; + HGC_options->set("b-dir", "direction of b", verbosity, B_DIR); + + int z = 0; + HGC_options->set("z-len", "length of z", verbosity, z); + + int nStout; + HGC_options->set("n_stout", "n parameter stout smearing", verbosity, nStout); + + double rhoStout; + HGC_options->set("rho_stout", "Rho parameter stout smearing", verbosity, rhoStout); + + size_t maxStout; + HGC_options->set("max_stout", "Maximum number of stout smearing steps", verbosity, maxStout); + + size_t stepStout; + HGC_options->set("step_stout", "Save the PDFs every step_stout stout smearing step", verbosity, stepStout); + + bool calc3pt = true ; + HGC_options->set("calc3pt", "If true then the 3pt function is computed", verbosity, calc3pt); + + std::string proj ; + HGC_options->set("which_projector", "Which projector to use for 3pt function", verbosity, proj); + WHICHPROJECTOR which_proj=get_projector(proj.c_str()); + + std::string conf; + HGC_options->set("conf", "configuration", verbosity, conf); + + initializePLEGMA(); + + // Reading from Lime file and loading to device + PLEGMA_Gauge gauge; + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + + // Loading to QUDA and computing plaquette also there + initGaugeQuda(gauge, true, QUDA_WILSON_LINKS); + plaqQuda(); + + WHICHPARTICLE nucleon = which_particle; + if(nucleon!=NEUTRON && nucleon!=PROTON) PLEGMA_error("Only nucleon PDFs have been implemented so far\n"); + + // Smearing + PLEGMA_Gauge smearedGauge; + smearedGauge.APEsmearing(gauge, nsmearAPE, alphaAPE, 3); + PLEGMA_printf("Plaquette after smearing:\n"); + smearedGauge.calculatePlaq(); + + //Momentum smearing: put the momentum phase to smeared gauge field + std::complex momSmScale[N_DIMS]; + std::complex I(0,1); + for(int i = 0 ; i < N_DIMS; i++) momSmScale[i] = std::exp(-(xiMomSm*2.*PI*sinkMom[i]/HGC_totalL[i])*I); + smearedGauge.scaleDirWise(momSmScale); + + // Extracting the spacial sink momentum from the sink 4-momentum + std::vector sinkMom_3D(3); + std::copy(sinkMom.begin(),sinkMom.begin()+3,sinkMom_3D.begin()); + + // ensuring mu positive + if(mu<0) mu*=-1.; + QUDA_solver *solver = new QUDA_solver(mu); + + PLEGMA_Vector vectorIn; + PLEGMA_Vector vectorOut; + PLEGMA_Vector vectorAuxD; + PLEGMA_Vector vectorAuxF; + PLEGMA_Propagator *propUP = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propDN = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *seqPropOut = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propIn = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propExchange = nullptr; + + PLEGMA_Gauge gaugeWL; + gaugeWL.copy(gauge); + gaugeWL.stoutSmearing(gaugeWL,nStout,rhoStout,3); + PLEGMA_Su3field *WL = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field tmp(BOTH); + PLEGMA_Su3field *u_s[4]; + for(int idir = 0; idir < 4 ; idir++){ + u_s[idir] = new PLEGMA_Su3field(BOTH); + u_s[idir]->absorbDir_device(gaugeWL,idir); + } + PLEGMA_Su3field *WLIn = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field *WLExchange = nullptr; + + PLEGMA_Su3field ZE(BOTH); + PLEGMA_Su3field tmp1(BOTH); + + PLEGMA_Propagator3D propUP3D; + PLEGMA_Propagator3D propDN3D; + + for(int isource=0;isource<=7;isource++){ + + PLEGMA_printf("SOURCE POSITION %d : %d %d %d %d \n",isource,sourcePositions[isource][0],sourcePositions[isource][1],sourcePositions[isource][2],sourcePositions[isource][3]); + + PLEGMA_Correlator corrThrpWL(corr_space, sourcePositions[isource], 0); + + for(int ts=0;ts= HGC_totalL[3] ? -1 : +1; + int global_fixSinkTime = (tSinks[ts] + sourcePositions[isource][3])%HGC_totalL[3]; + int my_fixSinkTime = global_fixSinkTime - HGC_procPosition[3] * HGC_localL[3]; + bool is_myST = (my_fixSinkTime >= 0) && ( my_fixSinkTime < HGC_localL[3] ); + + if(mu<0) { + mu*=-1.; + solver->UpdateSolver(); + } + for(int isc = 0 ; isc < 12 ; isc++){ + vectorAuxD.pointSource(sourcePositions[isource], isc/3, isc%3, DEVICE); + vectorIn.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + + PLEGMA_printf("Going to invert UP for component %d\n", isc); + solver->solve(vectorOut, vectorIn); + vectorAuxF.copy(vectorOut); + propUP->absorb(vectorAuxF, isc/3, isc%3); + vectorAuxD.gaussianSmearing(vectorOut, smearedGauge , nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorAuxD); + propUP3D.absorb(vectorAuxF,global_fixSinkTime, isc/3, isc%3); + } + + if(mu>0) { + mu*=-1.; + solver->UpdateSolver(); + } + for(int isc = 0 ; isc < 12 ; isc++){ + vectorAuxD.pointSource(sourcePositions[isource], isc/3, isc%3, DEVICE); + vectorIn.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + + PLEGMA_printf("Going to invert DN for component %d\n", isc); + solver->solve(vectorOut, vectorIn); + vectorAuxF.copy(vectorOut); + propDN->absorb(vectorAuxF, isc/3, isc%3); + vectorAuxD.gaussianSmearing(vectorOut, smearedGauge , nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorAuxD); + propDN3D.absorb(vectorAuxF,global_fixSinkTime, isc/3, isc%3); + } + + if(calc3pt){ + + propUP->unload(); + propDN->unload(); + //seq source part 2Props and contraction block + { + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + PLEGMA_Vector3D vectorAux3D; + if(nucleon == PROTON) + vectorAux3D.seqSourceNucleon(propUP3D, propDN3D, which_proj, nucleon, nu, c2); + else + vectorAux3D.seqSourceNucleon(propDN3D, propUP3D, which_proj, nucleon, nu, c2); + vectorAux3D.mulMomentumPhases(sinkMom,-1); // put momentum at the sink + std::complex Isingle(0,1); + float phase = 2.*PI*(((float) sinkMom[0] * sourcePositions[isource][0])/HGC_totalL[0] + + ((float)sinkMom[1] * sourcePositions[isource][1])/HGC_totalL[1] + + ((float)sinkMom[2] * sourcePositions[isource][2])/HGC_totalL[2]); + vectorAux3D.cscale(std::exp(+phase*Isingle)); // put momentum from the point source + vectorAux3D.conjugate(); + vectorAux3D.apply_gamma(G5); + vectorAuxF.absorb(vectorAux3D, global_fixSinkTime); + vectorAuxD.copy(vectorAuxF); + vectorIn.gaussianSmearing(vectorAuxD,smearedGauge, nsmearGauss, alphaGauss); + // check if we need to normalize the seqsource for mix precision solver + if(nucleon == PROTON){ + if(mu>0) { + mu*=-1.; + solver->UpdateSolver(); + } + } + else{ + if(mu<0) { + mu*=-1.; + solver->UpdateSolver(); + } + } + double norm = vectorIn.norm(); + vectorIn.cscale(1/norm); + solver->solve(vectorOut, vectorIn); + vectorOut.cscale(norm); + vectorAuxF.copy(vectorOut); + seqPropOut->absorb(vectorAuxF, nu, c2); + } + seqPropOut->apply_gamma(G5); + seqPropOut->conjugate(); + + int signProps = (nucleon == PROTON) ? +1: -1; + + PLEGMA_Propagator *propF = (nucleon == PROTON) ? propUP : propDN; + + for(int z_step=0 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+L_DIR)%8); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + } + + std::string suff = "_CP2_stout_"+std::to_string(nStout); + corrThrpWL.contractNucleonThrp_staple(*seqPropOut, *propF, *WL, signProps, gammas, false, b, l, z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL.getTotalSize()*2; iv++) corrThrpWL.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL.writeFile((threep_filename+"_CP2").c_str(), corr_file_format); + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0 ; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, L_DIR); + } + + for(int z_step=1 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,(4+L_DIR)%8); + } + } + + std::string suff = "_CP2_stout_"+std::to_string(nStout); + corrThrpWL.contractNucleonThrp_staple(*seqPropOut, *propF, *WL, signProps, gammas, false, b, -l, -z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL.getTotalSize()*2; iv++) corrThrpWL.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL.writeFile((threep_filename+"_CP2").c_str(), corr_file_format); + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0 ; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + } + + //seq source part 1Props and contraction block + { + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + PLEGMA_Vector3D vectorAux3D; + if(nucleon == PROTON) + vectorAux3D.seqSourceNucleon(propUP3D, which_proj, nucleon, nu, c2); + else + vectorAux3D.seqSourceNucleon(propDN3D, which_proj, nucleon, nu, c2); + vectorAux3D.mulMomentumPhases(sinkMom,-1); // put momentum at the sink + std::complex Isingle(0,1); + float phase = 2.*PI*(((float) sinkMom[0] * sourcePositions[isource][0])/HGC_totalL[0] + + ((float)sinkMom[1] * sourcePositions[isource][1])/HGC_totalL[1] + + ((float)sinkMom[2] * sourcePositions[isource][2])/HGC_totalL[2]); + vectorAux3D.cscale(std::exp(+phase*Isingle)); // put momentum from the point source + vectorAux3D.conjugate(); + vectorAux3D.apply_gamma(G5); + vectorAuxF.absorb(vectorAux3D, global_fixSinkTime); + vectorAuxD.copy(vectorAuxF); + vectorIn.gaussianSmearing(vectorAuxD,smearedGauge, nsmearGauss, alphaGauss); + + if(nucleon == PROTON){ + if(mu<0) { + mu*=-1.; + solver->UpdateSolver(); + } + } + else{ + if(mu>0) { + mu*=-1.; + solver->UpdateSolver(); + } + } + double norm = vectorIn.norm(); + vectorIn.cscale(1/norm); + solver->solve(vectorOut, vectorIn); + vectorOut.cscale(norm); + vectorAuxF.copy(vectorOut); + seqPropOut->absorb(vectorAuxF, nu, c2); + } + seqPropOut->apply_gamma(G5); + seqPropOut->conjugate(); + + int signProps = (nucleon == PROTON) ? -1: +1; + + PLEGMA_Propagator *propF = (nucleon == PROTON) ? propDN : propUP; + + for(int z_step=0 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+L_DIR)%8); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + + } + + std::string suff = "_CP1_stout_"+std::to_string(nStout); + corrThrpWL.contractNucleonThrp_staple(*seqPropOut, *propF, *WL, signProps, gammas, false, b, l, z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL.getTotalSize()*2; iv++) corrThrpWL.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL.writeFile((threep_filename+"_CP1").c_str(), corr_file_format); + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0 ; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, L_DIR); + } + + for(int z_step=1 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, L_DIR); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,(4+L_DIR)%8); + } + + } + + std::string suff = "_CP1_stout_"+std::to_string(nStout); + corrThrpWL.contractNucleonThrp_staple(*seqPropOut, *propF, *WL, signProps, gammas, false, b, -l, -z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL.getTotalSize()*2; iv++) corrThrpWL.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL.writeFile((threep_filename+"_CP1").c_str(), corr_file_format); + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + } + } + } + + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + vectorAuxF.absorb(*propUP, nu, c2); + vectorAuxD.copy(vectorAuxF); + vectorOut.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorOut); + propUP->absorb(vectorAuxF, nu, c2); + + vectorAuxF.absorb(*propDN, nu, c2); + vectorAuxD.copy(vectorAuxF); + vectorOut.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorOut); + propDN->absorb(vectorAuxF, nu, c2); + } + + propUP->rotateToPhysicalBase_device(+1); + propDN->rotateToPhysicalBase_device(-1); + propUP->applyBoundaries_device(sourcePositions[isource][3]); + propDN->applyBoundaries_device(sourcePositions[isource][3]); + + PLEGMA_Correlator corr(corr_space, sourcePositions[isource]); + corr.setFixMomVec(sinkMom_3D); + corr.contractMesons(*propUP, *propDN); + corr.writeFile(twop_filename.c_str(), corr_file_format); + + //!!!!!!!!!! maybe later we choose the specific momentum when this allows it + corr.contractBaryons(*propUP, *propDN); + corr.writeFile(twop_filename.c_str(), corr_file_format); + } + + for(int b=0;b<=B_LEN;b++){ + for(int l=0;l<=L_LEN;l++){ + + std::string namefile = "/onyx/qdata/jtarello/TMDPDF/ZE_b"+std::to_string(b)+"_l"+std::to_string(l)+"_stout"+std::to_string(nStout)+"_"+conf+".h5"; + + ZE.setUnit((std::vector) {0,4,8}); + + if(l!=0 || b!=0){ + int sspath[(4*l)+(2*b)]; + + for(int j=0;j vsspath(sspath,sspath+(4*l)+(2*b)); + + ZE.path(vsspath, u_s, tmp1); + } + + ZE.conjugate(); + + PLEGMA_Field ZEcontracted(BOTH,SCALAR); + + ZEcontracted.SU3Trace(ZE); + + ZEcontracted.writeFile(namefile,corr_file_format); + + } + } + + delete propUP; + delete propDN; + delete propIn; + delete seqPropOut; + + for(int idir = 0; idir < 4 ; idir++){ + delete u_s[idir]; + } + delete WL; + delete WLIn; + + delete solver; + + finalize(); + return 0; +} + diff --git a/plegma/TMDPDFs_pion.cpp b/plegma/TMDPDFs_pion.cpp new file mode 100644 index 00000000..b929173d --- /dev/null +++ b/plegma/TMDPDFs_pion.cpp @@ -0,0 +1,517 @@ +#include +#include + +using namespace plegma; +using namespace quda; + +int main(int argc, char **argv) +{ + + static std::vector listOpt = {"verbosity", "load-gauge", "nsmear-APE", "alpha-APE", "nsmear-gauss", "alpha-gauss", + "nsrc", "src-filename", "maxQsq", "twop-filename","threep-filename", "corr-file-format", + "corr-space", "tSinks","xiMomSm","sinkMom","gammas"}; + + initializeOptions(argc, argv, true, listOpt); + + if(mu<0){ + mu *= -1.0; + } + + double mu_ud = mu; + + double mu_s; + HGC_options->set("mu-s", "mu for strange quark", verbosity, mu_s); + + int L_LEN; + HGC_options->set("l-len", "length of l", verbosity, L_LEN); + + int L_DIR; + HGC_options->set("l-dir", "direction of l", verbosity, L_DIR); + + int B_LEN; + HGC_options->set("b-len", "length of b", verbosity, B_LEN); + + int B_DIR; + HGC_options->set("b-dir", "direction of b", verbosity, B_DIR); + + int z = 0; + HGC_options->set("z-len", "length of z", verbosity, z); + + int nStout; + HGC_options->set("n_stout", "n parameter stout smearing", verbosity, nStout); + + double rhoStout; + HGC_options->set("rho_stout", "Rho parameter stout smearing", verbosity, rhoStout); + + bool calc3pt = true ; + HGC_options->set("calc3pt", "If true then the 3pt function is computed", verbosity, calc3pt); + + std::string conf; + HGC_options->set("conf", "configuration", verbosity, conf); + + std::string twop_filename_1; + HGC_options->set("twop-filename-1", "2p filename for kaon +", verbosity, twop_filename_1); + + std::string threep_filename_1; + HGC_options->set("threep-filename-1", "3p filename for kaon +", verbosity, threep_filename_1); + + initializePLEGMA(); + + // Reading from Lime file and loading to device + PLEGMA_Gauge gauge; + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + + // Loading to QUDA and computing plaquette also there + initGaugeQuda(gauge, true, QUDA_WILSON_LINKS); + plaqQuda(); + + // Smearing + PLEGMA_Gauge smearedGauge; + PLEGMA_Gauge smearedGaugeMn; + smearedGauge.APEsmearing(gauge, nsmearAPE, alphaAPE, 3); + PLEGMA_printf("Plaquette after smearing:\n"); + smearedGauge.calculatePlaq(); + smearedGaugeMn.copy(smearedGauge); + + //Momentum smearing: put the momentum phase to smeared gauge field + std::complex momSmScale[N_DIMS]; + std::complex momSmScaleMn[N_DIMS]; + std::complex I(0,1); + for(int i = 0 ; i < N_DIMS; i++){ + momSmScale[i] = std::exp(-(xiMomSm*2.*PI*sinkMom[i]/HGC_totalL[i])*I); + momSmScaleMn[i] = std::exp(+(xiMomSm*2.*PI*sinkMom[i]/HGC_totalL[i])*I); + } + smearedGauge.scaleDirWise(momSmScale); + smearedGaugeMn.scaleDirWise(momSmScaleMn); + + // Extracting the spacial sink momentum from the sink 4-momentum + std::vector sinkMom_3D(3); + std::copy(sinkMom.begin(),sinkMom.begin()+3,sinkMom_3D.begin()); + + // ensuring mu positive + if(mu<0) mu*=-1.; + updateOptions(LIGHT); + QUDA_solver *solver = new QUDA_solver(mu); + + PLEGMA_Vector vectorIn; + PLEGMA_Vector vectorOut; + PLEGMA_Vector vectorAuxD; + PLEGMA_Vector vectorAuxF; + PLEGMA_Propagator *propUP_smearplus = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propUP_smearminus = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propST_smearminus = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *seqPropOut1 = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *seqPropOut2 = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propIn = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propExchange = nullptr; + + PLEGMA_Gauge gaugeWL; + gaugeWL.copy(gauge); + gaugeWL.stoutSmearing(gaugeWL,nStout,rhoStout,3); + PLEGMA_Su3field *WL = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field tmp(BOTH); + PLEGMA_Su3field *u_s[4]; + for(int idir = 0; idir < 4 ; idir++){ + u_s[idir] = new PLEGMA_Su3field(BOTH); + u_s[idir]->absorbDir_device(gaugeWL,idir); + } + PLEGMA_Su3field *WLIn = new PLEGMA_Su3field(BOTH); + PLEGMA_Su3field *WLExchange = nullptr; + + PLEGMA_Su3field ZE(BOTH); + PLEGMA_Su3field tmp1(BOTH); + + PLEGMA_Propagator3D propUP3D_smearplus; + PLEGMA_Propagator3D propUP3D_smearminus; + PLEGMA_Propagator3D propST3D_smearminus; + + for(int isource=0;isource corrThrpWL1(corr_space, sourcePositions[isource], 0); + PLEGMA_Correlator corrThrpWL2(corr_space, sourcePositions[isource], 0); + + for(int ts=0;ts= HGC_totalL[3] ? -1 : +1; + int global_fixSinkTime = (tSinks[ts] + sourcePositions[isource][3])%HGC_totalL[3]; + int my_fixSinkTime = global_fixSinkTime - HGC_procPosition[3] * HGC_localL[3]; + bool is_myST = (my_fixSinkTime >= 0) && ( my_fixSinkTime < HGC_localL[3] ); + + if(mu!=mu_ud) { + updateOptions(LIGHT); + mu=mu_ud; + solver->UpdateSolver(); + } + for(int isc = 0 ; isc < 12 ; isc++){ + vectorAuxD.pointSource(sourcePositions[isource], isc/3, isc%3, DEVICE); + vectorIn.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + + PLEGMA_printf("Going to invert UP for component %d\n", isc); + solver->solve(vectorOut, vectorIn); + vectorAuxF.copy(vectorOut); + propUP_smearplus->absorb(vectorAuxF, isc/3, isc%3); + vectorAuxD.gaussianSmearing(vectorOut, smearedGauge , nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorAuxD); + propUP3D_smearplus.absorb(vectorAuxF,global_fixSinkTime, isc/3, isc%3); + } + + for(int isc = 0 ; isc < 12 ; isc++){ + vectorAuxD.pointSource(sourcePositions[isource], isc/3, isc%3, DEVICE); + vectorIn.gaussianSmearing(vectorAuxD, smearedGaugeMn, nsmearGauss, alphaGauss); + + PLEGMA_printf("Going to invert UP for component %d\n", isc); + solver->solve(vectorOut, vectorIn); + vectorAuxF.copy(vectorOut); + propUP_smearminus->absorb(vectorAuxF, isc/3, isc%3); + vectorAuxD.gaussianSmearing(vectorOut, smearedGaugeMn , nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorAuxD); + propUP3D_smearminus.absorb(vectorAuxF,global_fixSinkTime, isc/3, isc%3); + } + + if(mu!=mu_s) { + updateOptions(STRANGE); + mu=mu_s; + solver->UpdateSolver(); + } + for(int isc = 0 ; isc < 12 ; isc++){ + vectorAuxD.pointSource(sourcePositions[isource], isc/3, isc%3, DEVICE); + vectorIn.gaussianSmearing(vectorAuxD, smearedGaugeMn, nsmearGauss, alphaGauss); + + PLEGMA_printf("Going to invert STRANGE for component %d\n", isc); + solver->solve(vectorOut, vectorIn); + vectorAuxF.copy(vectorOut); + propST_smearminus->absorb(vectorAuxF, isc/3, isc%3); + vectorAuxD.gaussianSmearing(vectorOut, smearedGaugeMn , nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorAuxD); + propST3D_smearminus.absorb(vectorAuxF,global_fixSinkTime, isc/3, isc%3); + } + + if(calc3pt){ + + propUP_smearplus->unload(); + propUP_smearminus->unload(); + propST_smearminus->unload(); + + //seq source part 2Props and contraction block for Pion + and Kaon + + { + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + PLEGMA_Vector3D vectorAux3D; + vectorAux3D.absorb(propUP3D_smearminus,nu,c2); + vectorAux3D.mulMomentumPhases(sinkMom,+1); //put momentum at the sink + std::complex Isingle(0,1); + float phase = 2.*PI*(((float) sinkMom[0] * sourcePositions[isource][0])/HGC_totalL[0] + + ((float)sinkMom[1] * sourcePositions[isource][1])/HGC_totalL[1] + + ((float)sinkMom[2] * sourcePositions[isource][2])/HGC_totalL[2]); + vectorAux3D.cscale(std::exp(-phase*Isingle)); // put momentum from the point source + vectorAux3D.apply_gamma(G5); + vectorAuxF.absorb(vectorAux3D, global_fixSinkTime); + vectorAuxD.copy(vectorAuxF); + vectorIn.gaussianSmearing(vectorAuxD,smearedGauge, nsmearGauss, alphaGauss); + // check if we need to normalize the seqsource for mix precision solver + if(mu!=-mu_ud) { + updateOptions(LIGHT); + mu=-mu_ud; + solver->UpdateSolver(); + } + double norm = vectorIn.norm(); + vectorIn.cscale(1/norm); + solver->solve(vectorOut, vectorIn); + vectorOut.cscale(norm); + vectorAuxF.copy(vectorOut); + seqPropOut1->absorb(vectorAuxF, nu, c2); + } + seqPropOut1->apply_gamma(G5); + seqPropOut1->conjugate(); + + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + PLEGMA_Vector3D vectorAux3D; + vectorAux3D.absorb(propST3D_smearminus,nu,c2); + vectorAux3D.mulMomentumPhases(sinkMom,+1); //put momentum at the sink + std::complex Isingle(0,1); + float phase = 2.*PI*(((float) sinkMom[0] * sourcePositions[isource][0])/HGC_totalL[0] + + ((float)sinkMom[1] * sourcePositions[isource][1])/HGC_totalL[1] + + ((float)sinkMom[2] * sourcePositions[isource][2])/HGC_totalL[2]); + vectorAux3D.cscale(std::exp(-phase*Isingle)); // put momentum from the point source + vectorAux3D.apply_gamma(G5); + vectorAuxF.absorb(vectorAux3D, global_fixSinkTime); + vectorAuxD.copy(vectorAuxF); + vectorIn.gaussianSmearing(vectorAuxD,smearedGauge, nsmearGauss, alphaGauss); + // check if we need to normalize the seqsource for mix precision solver + if(mu!=-mu_ud) { + updateOptions(LIGHT); + mu=-mu_ud; + solver->UpdateSolver(); + } + double norm = vectorIn.norm(); + vectorIn.cscale(1/norm); + solver->solve(vectorOut, vectorIn); + vectorOut.cscale(norm); + vectorAuxF.copy(vectorOut); + seqPropOut2->absorb(vectorAuxF, nu, c2); + } + seqPropOut2->apply_gamma(G5); + seqPropOut2->conjugate(); + + PLEGMA_Propagator *propF = propUP_smearplus; + + for(int z_step=0 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+L_DIR)%8); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,L_DIR); + } + } + + std::string suff = "_CP2_stout_"+std::to_string(nStout); + corrThrpWL1.contractNucleonThrp_staple(*seqPropOut1, *propF, *WL, +1, gammas, false, b, l, z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL1.getTotalSize()*2; iv++) corrThrpWL1.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL1.writeFile((threep_filename+"_CP2").c_str(), corr_file_format); + + corrThrpWL2.contractNucleonThrp_staple(*seqPropOut2, *propF, *WL, +1, gammas, false, b, l, z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL2.getTotalSize()*2; iv++) corrThrpWL2.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL2.writeFile((threep_filename_1+"_CP2").c_str(), corr_file_format); + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0 ; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, L_DIR); + } + + for(int z_step=1 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,L_DIR); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,(4+L_DIR)%8); + } + } + + std::string suff = "_CP2_stout_"+std::to_string(nStout); + corrThrpWL1.contractNucleonThrp_staple(*seqPropOut1, *propF, *WL, +1, gammas, false, b, -l, -z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL1.getTotalSize()*2; iv++) corrThrpWL1.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL1.writeFile((threep_filename+"_CP2").c_str(), corr_file_format); + + corrThrpWL2.contractNucleonThrp_staple(*seqPropOut2, *propF, *WL, +1, gammas, false, b, -l, -z_step); + if(signPer < 0) for(int iv = 0 ; iv < corrThrpWL2.getTotalSize()*2; iv++) corrThrpWL2.H_elem()[iv] *= signPer; + //corrThrpWL.writeASCII( (threep_filename + suff + "_ts_" + std::to_string(tSinks[ts]) + ".dat").c_str() ); + corrThrpWL2.writeFile((threep_filename_1+"_CP2").c_str(), corr_file_format); + + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, B_DIR); + } + + } + + for(int z_step=0 ; z_step < z ; z_step++){ + propExchange = propIn; propIn = propF; propF = propExchange; + propF->shift(*propIn, (4+L_DIR)%8); + } + + } + } + } + + for(int nu = 0 ; nu < 4 ; nu++) + for(int c2 = 0 ; c2 < 3 ; c2++){ + vectorAuxF.absorb(*propUP_smearplus, nu, c2); + vectorAuxD.copy(vectorAuxF); + vectorOut.gaussianSmearing(vectorAuxD, smearedGauge, nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorOut); + propUP_smearplus->absorb(vectorAuxF, nu, c2); + + vectorAuxF.absorb(*propUP_smearminus, nu, c2); + vectorAuxD.copy(vectorAuxF); + vectorOut.gaussianSmearing(vectorAuxD, smearedGaugeMn, nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorOut); + propUP_smearminus->absorb(vectorAuxF, nu, c2); + + vectorAuxF.absorb(*propST_smearminus, nu, c2); + vectorAuxD.copy(vectorAuxF); + vectorOut.gaussianSmearing(vectorAuxD, smearedGaugeMn, nsmearGauss, alphaGauss); + vectorAuxF.copy(vectorOut); + propST_smearminus->absorb(vectorAuxF, nu, c2); + + } + + propUP_smearplus->rotateToPhysicalBase_device(+1); + propUP_smearminus->rotateToPhysicalBase_device(+1); + propST_smearminus->rotateToPhysicalBase_device(+1); + propUP_smearplus->applyBoundaries_device(sourcePositions[isource][3]); + propUP_smearminus->applyBoundaries_device(sourcePositions[isource][3]); + propST_smearminus->applyBoundaries_device(sourcePositions[isource][3]); + + PLEGMA_Correlator corr(corr_space, sourcePositions[isource]); + corr.setFixMomVec(sinkMom_3D); + corr.contractMesonsNew(*propUP_smearplus, *propUP_smearminus); + corr.writeFile(twop_filename.c_str(), corr_file_format); + + PLEGMA_Correlator corr1(corr_space, sourcePositions[isource]); + corr1.setFixMomVec(sinkMom_3D); + corr1.contractMesonsNew(*propUP_smearplus, *propST_smearminus); + corr1.writeFile(twop_filename_1.c_str(), corr_file_format); + + } + + /***for(int b=0;b<=B_LEN;b++){ + for(int l=0;l<=L_LEN;l++){ + + std::string namefile = "/onyx/qdata/jtarello/TMDPDFs_pion/ZE_b"+std::to_string(b)+"_l"+std::to_string(l)+"_stout"+std::to_string(nStout)+"_"+conf+".h5"; + + ZE.setUnit((std::vector) {0,4,8}); + + if(l!=0 || b!=0){ + int sspath[(4*l)+(2*b)]; + + for(int j=0;j vsspath(sspath,sspath+(4*l)+(2*b)); + + ZE.path(vsspath, u_s, tmp1); + } + + ZE.conjugate(); + + PLEGMA_Field ZEcontracted(BOTH,SCALAR); + + ZEcontracted.SU3Trace(ZE); + + PLEGMA_FT ZEFT((std::vector) {0,0,0},3); + + ZEFT.apply(ZEcontracted); + + ZEFT.writeFile(namefile,corr_file_format); + + } + }***/ + + delete propUP_smearplus; + delete propUP_smearminus; + delete propST_smearminus; + + delete propIn; + delete seqPropOut1; + delete seqPropOut2; + + for(int idir = 0; idir < 4 ; idir++){ + delete u_s[idir]; + } + delete WL; + delete WLIn; + + delete solver; + + finalize(); + return 0; +} + diff --git a/plegma/WallSourceNucleon.cpp b/plegma/WallSourceNucleon.cpp new file mode 100644 index 00000000..94db2dcc --- /dev/null +++ b/plegma/WallSourceNucleon.cpp @@ -0,0 +1,145 @@ +#include +#include + +using namespace plegma; +using namespace quda; + +int main(int argc, char **argv) +{ + + static std::vector listOpt = {"verbosity", "load-gauge", "nsmear-APE", "alpha-APE", "nsmear-gauss", "alpha-gauss", + "nsrc", "src-filename", "maxQsq", "twop-filename","threep-filename", "corr-file-format", + "corr-space", "tSinks","Projs","xiMomSm","sinkMom","which_particle","gammas"}; + + initializeOptions(argc, argv, true, listOpt); + + double mu_ud; + + int tsrc; + HGC_options->set("t-source", "Time of source vector", verbosity, tsrc); + + std::string ft1_filename; + std::string ft2_filename; + std::string ft3_filename; + std::string ft4_filename; + HGC_options->set("ft1-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft1_filename); + HGC_options->set("ft2-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft2_filename); + HGC_options->set("ft3-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft3_filename); + HGC_options->set("ft4-filename", "If added to the momentum transfer delta gives the sink momentum", verbosity, ft4_filename); + + initializePLEGMA(); + + std::vector zero_mom = {0,0,0}; + + PLEGMA_Gauge gauge; + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + + initGaugeQuda(gauge, true, QUDA_WILSON_LINKS); + plaqQuda(); + + PLEGMA_Gauge smearedGauge; + smearedGauge.APEsmearing(gauge, nsmearAPE, alphaAPE, 3); + PLEGMA_printf("Plaquette after smearing:\n"); + smearedGauge.calculatePlaq(); + + if(mu<0) mu*=-1.; + mu_ud = mu; + QUDA_solver *solver = new QUDA_solver(mu); + + PLEGMA_Vector vectorIn; + PLEGMA_Vector vectorOut; + PLEGMA_Vector vectorAuxD; + PLEGMA_Vector vectorAuxF; + + PLEGMA_Propagator *propUPp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propDNp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propUPPp = new PLEGMA_Propagator(BOTH); + PLEGMA_Propagator *propDNPp = new PLEGMA_Propagator(BOTH); + + site& source = sourcePositions[0]; + + if(mu != mu_ud) { + mu = mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propUPp->absorb(vectorAuxF, isc/3, isc%3); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + vectorIn.mulMomentumPhases((std::vector) {sinkMom[0],sinkMom[1],sinkMom[2],0},+1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propUPPp->absorb(vectorAuxF, isc/3, isc%3); + } + + propUPPp->rotateToPhysicalBase_device(+1); + propUPp->rotateToPhysicalBase_device(+1); + propUPPp->applyBoundaries_device(0); + propUPp->applyBoundaries_device(0); + + PLEGMA_FT ftUPp(zero_mom,3); + PLEGMA_FT ftUPPp(sinkMom,3); + ftUPp.apply(*propUPp); + ftUPPp.apply(*propUPPp); + ftUPp.writeFile(ft1_filename, corr_file_format); + ftUPPp.writeFile(ft2_filename, corr_file_format); + + if(mu != -mu_ud) { + mu = -mu_ud; + solver->UpdateSolver(); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propDNp->absorb(vectorAuxF, isc/3, isc%3); + } + + for(int isc=0;isc<12;isc++){ + vectorAuxD.setUnit((std::vector) {isc}); + vectorIn.absorbTimeslice(vectorAuxD,tsrc,true); + vectorIn.mulMomentumPhases((std::vector) {sinkMom[0],sinkMom[1],sinkMom[2],0},+1); + solver->solve(vectorOut,vectorIn); + vectorAuxF.copy(vectorOut); + propDNPp->absorb(vectorAuxF, isc/3, isc%3); + } + + propDNp->rotateToPhysicalBase_device(-1); + propDNp->applyBoundaries_device(source[0]); + propDNPp->rotateToPhysicalBase_device(-1); + propDNPp->applyBoundaries_device(source[0]); + + PLEGMA_FT ftDNp(zero_mom,3); + PLEGMA_FT ftDNPp(sinkMom,3); + ftDNp.apply(*propDNp); + ftDNPp.apply(*propDNPp); + ftDNp.writeFile(ft3_filename, corr_file_format); + ftDNPp.writeFile(ft4_filename, corr_file_format); + + PLEGMA_Correlator corr(corr_space, source); + corr.setFixMomVec(sinkMom); + corr.contractBaryonsWall(*propUPp,*propDNPp,*propDNp,*propUPPp); + corr.writeFile(twop_filename, corr_file_format); + + delete propUPp; + delete propUPPp; + delete propDNp; + delete propDNPp; + + delete solver; + + finalize(); + + return 0; +} diff --git a/plegma/Zfac.cpp b/plegma/Zfac.cpp index f645a39f..51f3a486 100644 --- a/plegma/Zfac.cpp +++ b/plegma/Zfac.cpp @@ -55,27 +55,25 @@ int main(int argc, char **argv) vecOut.apply_gamma(G4,LEFT); rprop.absorb(vecOut, isc/3, isc%3); // propagator in g0 convention in UKQCD (implicit flavor flip from g5g4) } - rprop.rotateToPhysicalBase_device(-1); // propagator in g0 convention in chiral (flipped flavor sign) + //rprop.rotateToPhysicalBase_device(-1); // propagator in g0 convention in chiral (flipped flavor sign) // compute the prop in momentum space PLEGMA_FT ft((std::vector) {(double)mom[0], (double)mom[1], (double)mom[2], mom[3]+0.5},4,false); // half twist in temporal direction // rprop.scale(1./HGC_totalVolume); ft.apply(rprop, FT_GEMV,-1); // rprop.scale(HGC_totalVolume); - ft.writeASCII(filesPrefix+"GpropMom_"+pxpypzpt+"_conf_"+confStr+".dat",0); + ft.writeASCII(filesPrefix+"zfac_GpropMom_"+pxpypzpt+"_conf_"+confStr+".dat",0); /////////// lprop.copy(rprop); - - // lprop.rotateToPhysicalBase_device(+1); // this needs to be in (1+ig5) // lprop.rotateToPhysicalBase_device(+1); - // lprop.apply_gamma(G5,LEFT); - // lprop.apply_gamma(G5,RIGHT); + lprop.apply_gamma(G5,LEFT); + lprop.apply_gamma(G5,RIGHT); // lprop.rotateToPhysicalBase_device(-1); - lprop.rotateToPhysicalBase_device(+1); - lprop.rotateToPhysicalBase_device(+1); + //lprop.rotateToPhysicalBase_device(+1); + //lprop.rotateToPhysicalBase_device(+1); lprop.conjugate(); // this is to make the dagger, trans is in the code site source({0,0,0,0}); @@ -83,16 +81,16 @@ int main(int argc, char **argv) PLEGMA_Correlator corr(MOMENTUM_SPACE, source, 0, HGC_totalL[3]); corr.contractNucleonThrp_local(lprop,rprop,0,gammas,true); - corr.writeFile(filesPrefix+"Vloc_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); + corr.writeFile(filesPrefix+"zfac_Vloc_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); - corr.contractNucleonThrp_oneD(lprop,rprop,gauge,0,gammas,true); - corr.writeFile(filesPrefix+"VoneD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); + /***corr.contractNucleonThrp_oneD(lprop,rprop,gauge,0,gammas,true); + corr.writeFile(filesPrefix+"zfac_VoneD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); corr.contractNucleonThrp_twoD(lprop,rprop,gauge,0,gammas,true); - corr.writeFile(filesPrefix+"VtwoD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); + corr.writeFile(filesPrefix+"zfac_VtwoD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); corr.contractNucleonThrp_threeD(lprop,rprop,gauge,0,gammas,true); - corr.writeFile(filesPrefix+"VthreeD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); + corr.writeFile(filesPrefix+"zfac_VthreeD_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT);***/ } diff --git a/plegma/ZfacStaple.cpp b/plegma/ZfacStaple.cpp new file mode 100644 index 00000000..6124bdb9 --- /dev/null +++ b/plegma/ZfacStaple.cpp @@ -0,0 +1,190 @@ +#include +#include + +using namespace plegma; +using namespace quda; +static std::vector listOpt = {"verbosity", "load-gauge", "nsrc","src-filename"}; +// we use src-filename to provide the momenta we want to do + +int main(int argc, char **argv) +{ + initializeOptions(argc, argv, true, listOpt); + //================ Add your options in this between initializeOptions and initializePLEGMA ================// + int L_LEN; + HGC_options->set("l-len", "length of l", verbosity, L_LEN); + int L_DIR; + HGC_options->set("l-dir", "direction of l", verbosity, L_DIR); + int B_LEN; + HGC_options->set("b-len", "length of b", verbosity, B_LEN); + int B_DIR; + HGC_options->set("b-dir", "direction of b", verbosity, B_DIR); + int z; + HGC_options->set("z-len", "length of z", verbosity, z); + int nStout; + HGC_options->set("n_stout", "n parameter stout smearing", verbosity, nStout); + double rhoStout; + HGC_options->set("rho_stout", "Rho parameter stout smearing", verbosity, rhoStout); + std::string boundaryCond = "antiperiodic"; + HGC_options->set("boundary-condition", "If we want periodic or antiperiodic in the temporal direction", verbosity, boundaryCond); + std::string filesPrefix="./"; + HGC_options->set("output-path", "Path to the directory to dump results", verbosity, filesPrefix); + //==========================// + initializePLEGMA(); + + // Read the configurations // + // Warning gauge fixing is needed // + PLEGMA_Gauge gauge(BOTH,FIRST_VERTEX); + gauge.readFile(latfile, LIME_FORMAT); + gauge.calculatePlaq(); + initGaugeQuda(gauge, boundaryCond == "antiperiodic"); + plaqQuda(); + std::string confStr=splitStrFwd(latfile,'.'); + if(boundaryCond == "antiperiodic") applyBoundaryConditions(gauge,true); + ///////////////////////////// + + if(mu < 0) mu = -mu; + QUDA_solver solver(mu); + PLEGMA_Vector vecIn; + PLEGMA_Vector vecOut; + PLEGMA_Propagator *rprop_d = new PLEGMA_Propagator(BOTH,FIRST_VERTEX); + PLEGMA_Propagator *lprop_d = new PLEGMA_Propagator(BOTH,FIRST_VERTEX); + PLEGMA_Propagator *propIn = new PLEGMA_Propagator(BOTH,FIRST_VERTEX); + PLEGMA_Propagator *propExchange = nullptr; + + PLEGMA_Gauge gaugeWL(BOTH,FIRST_VERTEX); + gaugeWL.copy(gauge); + gaugeWL.stoutSmearing(gaugeWL,nStout,rhoStout,3); + PLEGMA_Su3field *WL = new PLEGMA_Su3field(BOTH,FIRST_VERTEX); + PLEGMA_Su3field tmp(BOTH,FIRST_VERTEX); + PLEGMA_Su3field *u_s[4]; + for(int idir = 0; idir < 4 ; idir++){ + u_s[idir] = new PLEGMA_Su3field(BOTH,FIRST_VERTEX); + u_s[idir]->absorbDir_device(gaugeWL,idir); + } + PLEGMA_Su3field *WLIn = new PLEGMA_Su3field(BOTH,FIRST_VERTEX); + PLEGMA_Su3field *WLExchange = nullptr; + + site source({0,0,0,0}); + std::vector gammas = {ONE,G1,G2,G3,G4,G5,G5G1,G5G2,G5G3,G5G4,S12,S13,S23,S41,S42,S43}; + + PLEGMA_Correlator corr(MOMENTUM_SPACE, source, 0, HGC_totalL[3]); + + for(int i = 0; i < numSourcePositions; i++){ // iterate over the momenta we want to do + site& sS = sourcePositions[i]; + std::vector mom= {sS[0],sS[1],sS[2],sS[3]}; + std::string pxpypzpt = "px" + std::to_string(sS[0]) + "py" + std::to_string(sS[1]) + + "pz" + std::to_string(sS[2]) + "pt" + std::to_string(sS[3]); + // outName = pathOut + sourceType + "_" + pxpypzpt + "." + confStr + ".data"; + + if(mu<0) { + mu*=-1.; + solver.UpdateSolver(); + } + + for(int isc =0; isc < 12; isc++){ // compute the prop with mom source + std::vector scInd = {isc}; + vecIn.setUnit(scInd); + + vecIn.mulMomentumPhases((std::vector) {sS[0],sS[1],sS[2],sS[3]},+1); + // vecIn.rotate_uk_ch(); + if(boundaryCond == "antiperiodic") vecIn.mulThetaPhase(1.,true); + vecIn.apply_gamma(G4,LEFT); + vecIn.apply_gamma(G5,LEFT); + solver.solve(vecOut,vecIn); + vecOut.apply_gamma(G5,LEFT); + vecOut.apply_gamma(G4,LEFT); + rprop_d->absorb(vecOut, isc/3, isc%3); // propagator in g0 convention in UKQCD (implicit flavor flip from g5g4) + } + //rprop_d->rotateToPhysicalBase_device(-1); // propagator in g0 convention in chiral (flipped flavor sign) + + // compute the prop in momentum space + PLEGMA_FT ft((std::vector) {(double)mom[0], (double)mom[1], (double)mom[2], mom[3]+0.5},4,false); + ft.apply(*rprop_d, FT_GEMV,-1); + ft.writeASCII(filesPrefix+"new_GdpropMom_mu000533_twistbase_"+pxpypzpt+"_conf_"+confStr+".dat",0); + + /////////// + lprop_d->copy(*rprop_d); + + lprop_d->apply_gamma(G5,LEFT); + lprop_d->apply_gamma(G5,RIGHT); + + //lprop_d->rotateToPhysicalBase_device(-1); + //lprop_d->rotateToPhysicalBase_device(-1); + lprop_d->conjugate(); // this is to make the dagger, trans is in the code + + for(int z_step=0 ; z_step <= z ; z_step++){ + + if(z_step>0){ + propExchange = propIn; propIn = rprop_d; rprop_d = propExchange; + rprop_d->shift(*propIn, (4+L_DIR)%8); + } + + for(int b=0 ; b <= B_LEN ; b++){ + + if(b>0){ + propExchange = propIn; propIn = rprop_d; rprop_d = propExchange; + rprop_d->shift(*propIn, (4+B_DIR)%8); + } + + for(int l=0 ; l <= L_LEN ; l++){ + + WL->setUnit((std::vector) {0,4,8}); + + int spath[(2*l)+b+z_step]; + + if((l!=0)||(b!=0)||(z_step!=0)){ + for(int i=0;i vspath(spath,spath+(2*l)+b+z_step); + + WL->path(vspath, u_s, tmp); + + for(int k=(l+b);k<((2*l)+b+z_step);k++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+L_DIR)%8); + } + for(int j=l;j<(l+b);j++){ + WLExchange = WLIn; WLIn = WL; WL = WLExchange; + WL->shift(*WLIn,(4+B_DIR)%8); + } + for(int i=0;ishift(*WLIn,(L_DIR)%8); + } + + } + + corr.contractNucleonThrp_staple(*lprop_d, *rprop_d, *WL, 0, gammas, true, b, l, z_step); + corr.writeFile(filesPrefix+"new_Staple_mu000533_twistbase_ud_stout5_"+pxpypzpt+"_conf_"+confStr,HDF5_FORMAT); + + } + } + + for(int b=0 ; b < B_LEN ; b++){ + propExchange = propIn; propIn = rprop_d; rprop_d = propExchange; + rprop_d->shift(*propIn, B_DIR); + } + } + } + + delete rprop_d; + delete lprop_d; + delete propIn; + + for(int idir = 0; idir < 4 ; idir++){ + delete u_s[idir]; + } + delete WL; + delete WLIn; + + finalize(); + return 0; +} From 3084a48acf90b07d3b70cda90560e4fd531bbdfe Mon Sep 17 00:00:00 2001 From: Simone Bacchio Date: Thu, 13 Oct 2022 13:11:16 +0300 Subject: [PATCH 05/25] Delete .#WallSourceNucleon.shexit --- .#WallSourceNucleon.shexit | 1 - 1 file changed, 1 deletion(-) delete mode 120000 .#WallSourceNucleon.shexit diff --git a/.#WallSourceNucleon.shexit b/.#WallSourceNucleon.shexit deleted file mode 120000 index b3f13a0a..00000000 --- a/.#WallSourceNucleon.shexit +++ /dev/null @@ -1 +0,0 @@ -jtarello@cyclamen.4608:1636095945 \ No newline at end of file From d6bb238d6975e03cc48b0d307daa4188850052b0 Mon Sep 17 00:00:00 2001 From: Simone Bacchio Date: Thu, 13 Oct 2022 13:11:24 +0300 Subject: [PATCH 06/25] Delete .#slurm-243497.out --- .#slurm-243497.out | 1 - 1 file changed, 1 deletion(-) delete mode 120000 .#slurm-243497.out diff --git a/.#slurm-243497.out b/.#slurm-243497.out deleted file mode 120000 index a09050b6..00000000 --- a/.#slurm-243497.out +++ /dev/null @@ -1 +0,0 @@ -jtarello@cyclamen.21678:1651238169 \ No newline at end of file From 368f609174db183513fb72374aea5aa70a0492d5 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 18 Oct 2022 11:43:47 +0200 Subject: [PATCH 07/25] replacing checkCudaError with checkQudaError --- include/PLEGMA_BLAS.h | 4 +- include/PLEGMA_global.h | 3 + include/global/PLEGMA_prints.hpp | 3 +- lib/PLEGMA_Field.cu | 46 +-- lib/PLEGMA_Gauge.cu | 10 +- lib/PLEGMA_Propagator.cu | 14 +- lib/PLEGMA_QLoops.cu | 2 +- lib/PLEGMA_Random.cu | 6 +- lib/PLEGMA_Su3field.cu | 6 +- lib/PLEGMA_Vector.cu | 20 +- lib/kernels/PLEGMA_FT.cuh | 8 +- lib/kernels/PLEGMA_SU3_projection.cuh | 2 +- lib/kernels/PLEGMA_WFlow.cuh | 6 +- lib/kernels/PLEGMA_contractG5_bilinear.cuh | 2 +- lib/kernels/PLEGMA_covD.cuh | 2 +- lib/kernels/PLEGMA_field_utils.cuh | 8 +- lib/kernels/PLEGMA_fmunu_utils.cuh | 2 +- lib/kernels/PLEGMA_gFixing.cuh | 4 +- lib/kernels/PLEGMA_gammas_scatt.cuh | 2 +- lib/kernels/PLEGMA_gauge_utils.cuh | 4 +- lib/kernels/PLEGMA_kernel_tuner.cuh | 6 +- lib/kernels/PLEGMA_plaquette.cuh | 2 +- lib/kernels/PLEGMA_plaquetteCorners.cuh | 2 +- lib/kernels/PLEGMA_propagator_utils.cuh | 6 +- lib/kernels/PLEGMA_scattreductions.cuh | 60 ++-- lib/kernels/PLEGMA_scattreductionsPiPi.cuh | 4 +- lib/kernels/PLEGMA_seqSourceNucleon.cu | 2 +- lib/kernels/PLEGMA_shifts.cuh | 6 +- lib/kernels/PLEGMA_su3field.cuh | 17 +- lib/kernels/PLEGMA_threep_local_stochastic.cu | 184 ++++++++++++ lib/kernels/PLEGMA_threep_twoD_stochastic.cu | 264 ++++++++++++++++++ lib/kernels/PLEGMA_topocharge.cuh | 6 +- lib/kernels/PLEGMA_u1gauge_utils.cuh | 2 +- lib/kernels/PLEGMA_vector_utils.cuh | 18 +- plegma/qLoops.cpp | 2 +- plegma/qLoops_WilsonLine.cpp | 2 +- utils/PLEGMA_eigSolver.cpp | 8 +- utils/QUDA_params.cpp | 4 +- 38 files changed, 615 insertions(+), 134 deletions(-) create mode 100644 lib/kernels/PLEGMA_threep_local_stochastic.cu create mode 100644 lib/kernels/PLEGMA_threep_twoD_stochastic.cu diff --git a/include/PLEGMA_BLAS.h b/include/PLEGMA_BLAS.h index 12b15c89..b0e692db 100644 --- a/include/PLEGMA_BLAS.h +++ b/include/PLEGMA_BLAS.h @@ -233,7 +233,7 @@ namespace cuBLAS{ if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); cuBLAS::gemv_(trans, m, n, alpha, A, x, beta, y); cudaMemcpy(yHost,y,n*2*sizeof(Float),cudaMemcpyDeviceToHost); - checkCudaError(); + checkQudaError(); int mpiErr = MPI_Allreduce(MPI_IN_PLACE,yHost,n*2,MPI_Type(yHost),MPI_SUM,comm); if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); } @@ -245,7 +245,7 @@ namespace cuBLAS{ Float yHost[n*2]; cuBLAS::gemv(trans,m, n, alpha, A, x, beta, y, yHost,comm); cudaMemcpy(y,yHost,sizeof(yHost),cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); } } //=================================================================// diff --git a/include/PLEGMA_global.h b/include/PLEGMA_global.h index 02714678..74e883e0 100644 --- a/include/PLEGMA_global.h +++ b/include/PLEGMA_global.h @@ -43,6 +43,9 @@ using namespace std::chrono_literals; //======== Preprocessor macros =========// #include +#include +#include +using namespace quda; namespace plegma { //======== PLEGMA_printf, PLEGMA_error, PLEGMA_warning =========// diff --git a/include/global/PLEGMA_prints.hpp b/include/global/PLEGMA_prints.hpp index 86a8b07c..9c2011a6 100644 --- a/include/global/PLEGMA_prints.hpp +++ b/include/global/PLEGMA_prints.hpp @@ -3,12 +3,11 @@ * inspired by QUDA. */ #pragma once -#include + extern bool HGC_hold_exit; // used to hold exit until all the errors have been printed extern bool HGC_init_PLEGMA_flag; extern struct global_vars HGC_global_vars; extern class Options * HGC_options; -using namespace quda; #define PLEGMA_exit(value) do { \ if(! HGC_hold_exit) { \ if (HGC_init_PLEGMA_flag) { \ diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index cdf8ee4f..d487a745 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -204,14 +204,14 @@ template void PLEGMA_Field::load(){ if(allocation != BOTH) PLEGMA_error("Load from Host to Device needs BOTH allocation"); cudaMemcpy(d_elem, h_elem, Bytes_total(), cudaMemcpyHostToDevice ); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } template void PLEGMA_Field::unload() const{ if(allocation != BOTH) PLEGMA_error("Unload from Device to Host needs BOTH allocation"); cudaMemcpy(h_elem, d_elem, Bytes_total(), cudaMemcpyDeviceToHost); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } @@ -225,8 +225,9 @@ void PLEGMA_Field::create_host(){ template void PLEGMA_Field::create_device(){ - cudaMalloc((void**)&d_elem,Bytes_total_plus_ghost()); - if(checkErr) checkCudaError(); + d_elem=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, Bytes_total_plus_ghost()); +// cudaMalloc((void**)&d_elem,Bytes_total_plus_ghost()); + if(checkErr) checkQudaError(); #ifdef DEVICE_MEMORY_REPORT // device memory in MB HGC_deviceMemory += Bytes_total_plus_ghost()/(1024.*1024.); @@ -260,7 +261,7 @@ void PLEGMA_Field::create_device(){ hostMalloc(h_ext_ghost_vertex_s, Bytes_ghostVertex()); #endif } - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); isAllocDevice = true; } @@ -274,7 +275,7 @@ void PLEGMA_Field::destroy_host(){ template void PLEGMA_Field::destroy_device(){ cudaFree(d_elem); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); d_elem = NULL; #ifdef DEVICE_MEMORY_REPORT HGC_deviceMemory -= Bytes_total_plus_ghost()/(1024.*1024.); @@ -307,7 +308,7 @@ void PLEGMA_Field::destroy_device(){ hostFree(h_ext_ghost_vertex_s,Bytes_ghostVertex()); h_ext_ghost_vertex_s=NULL; #endif } - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); isAllocDevice=false; } @@ -430,7 +431,7 @@ void PLEGMA_Field::communicateSideGhost(short dir, ORIENTATION sign, ACTI size_t nbytes = HGC_surface3D[i]/scaleT*field_length*2*sizeof(Float); cudaMemcpy(pointer_send, pointer_device, nbytes, cudaMemcpyDeviceToHost); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); disp = (s==DIR_PLUS) ? +1 : -1; messages.push_back(comm_declare_receive_relative(pointer_receive,i,disp,nbytes)); @@ -451,7 +452,7 @@ void PLEGMA_Field::communicateSideGhost(short dir, ORIENTATION sign, ACTI Float *host = h_ext_ghost_r; Float *device = d_elem+total_length*field_length*2; cudaMemcpy(device, host, Bytes_ghost(),cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } else { for(short i=0; i::communicateSideGhost(short dir, ORIENTATION sign, ACTI Float *device = d_elem + (HGC_sideGhost[dir][s]/scaleT+total_length)*field_length*2; cudaMemcpy(device, host, HGC_surface3D[dir]/scaleT*field_length*2*sizeof(Float), cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } } } @@ -502,7 +503,7 @@ void PLEGMA_Field::communicateCornerGhost(short dir, ORIENTATION sign, AC size_t nbytes = HGC_surface2D[OFF2(i,j)]/scaleT*field_length*2*sizeof(Float); cudaMemcpy(pointer_send, pointer_device, nbytes, cudaMemcpyDeviceToHost); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); // communicating disp[i] = (s1==DIR_PLUS) ? +1 : -1; @@ -527,7 +528,7 @@ void PLEGMA_Field::communicateCornerGhost(short dir, ORIENTATION sign, AC Float *hostCorner = h_ext_ghost_corner_r; Float *device = d_elem+(total_length+ghost_length)*field_length*2; cudaMemcpy(device,hostCorner,Bytes_ghostCorner(),cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } else { for(short i=0; i::communicateCornerGhost(short dir, ORIENTATION sign, AC Float *device = d_elem+(HGC_cornerGhost[OFF2SIGN(i,j,s1,s2)]/scaleT+total_length+ghost_length)*field_length*2; cudaMemcpy(device, hostCorner, HGC_surface2D[OFF2(i,j)]/scaleT*field_length*2*sizeof(Float), cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } } } @@ -580,7 +581,7 @@ void PLEGMA_Field::communicateVertexGhost(short dir, ORIENTATION sign, AC size_t nbytes = HGC_surface1D[OFF3(i,j,k)]/scaleT*field_length*2*sizeof(Float); cudaMemcpy(pointer_send, pointer_device, nbytes, cudaMemcpyDeviceToHost); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); // communicating disp[i] = (s1==DIR_PLUS) ? +1 : -1; @@ -607,7 +608,7 @@ void PLEGMA_Field::communicateVertexGhost(short dir, ORIENTATION sign, AC Float *hostVertex = h_ext_ghost_vertex_r; Float *device = d_elem+(total_length+ghost_length)*field_length*2; cudaMemcpy(device,hostVertex,Bytes_ghostVertex(),cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } else { for(short i=0; i::communicateVertexGhost(short dir, ORIENTATION sign, AC Float *device = d_elem+(HGC_vertexGhost[OFF3SIGN(i,j,k,s1,s2,s3)]/scaleT+total_length+ghost_length+ghost_corner_length)*field_length*2; cudaMemcpy(device, hostVertex, HGC_surface1D[OFF3(i,j,k)]/scaleT*field_length*2*sizeof(Float), cudaMemcpyHostToDevice); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } } } @@ -670,14 +671,14 @@ template void PLEGMA_Field::shift(PLEGMA_Field &Fin, short dirOr1, short dirOr2, short dirOr3){ // we have to make sure that we have the ghost assert(dirOr1!=dirOr2 && dirOr2!=dirOr3 && dirOr1!=dirOr3); - Fin.communicateGhost(-1, DIR_BOTH, FIRST_VERTEX); + /Fin.communicateGhost(-1, DIR_BOTH, FIRST_VERTEX); shiftField(Fin,*this,dirOr1,dirOr2,dirOr3); } template void PLEGMA_Field::randInit(int seed){ randstate_ptr = new PLEGMA_RNG(seed, total_length); - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } template @@ -703,7 +704,7 @@ void PLEGMA_Field::stochastic_Z(int n){ default: PLEGMA_error("This value of n has not been compiled. Come here to add it"); } - if(checkErr) checkCudaError(); + if(checkErr) checkQudaError(); } template @@ -740,7 +741,8 @@ void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign) int D3D4 = mom.size(); int V = D3D4 == 3 ? HGC_localVolume3D : HGC_localVolume; Float2 *x; - cudaMalloc((void**)&x, V*2*sizeof(Float)); + x=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, V*2*sizeof(Float)); +// cudaMalloc((void**)&x, V*2*sizeof(Float)); cudaMemset((void*) x,0,V*2*sizeof(Float)); if(checkErr) checkQudaError(); std::vector momF(mom.begin(), mom.end()); @@ -870,7 +872,7 @@ void PLEGMA_Field::absorb(const PLEGMA_Field3D &field, int global_ pointer_dst = (this->D_elem() + i*V4 + my_it*V3); cudaMemcpy(pointer_dst, pointer_src, V3 * sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } template @@ -1023,7 +1025,7 @@ void PLEGMA_Field::absorbTimeslice(PLEGMA_Field &srcfield, int glo } } comm_barrier(); - checkCudaError(); + checkQudaError(); } diff --git a/lib/PLEGMA_Gauge.cu b/lib/PLEGMA_Gauge.cu index e2b49060..e79ca310 100644 --- a/lib/PLEGMA_Gauge.cu +++ b/lib/PLEGMA_Gauge.cu @@ -108,7 +108,7 @@ template void PLEGMA_Gauge::absorbDir_device(PLEGMA_Su3field &su,int dir){ cudaMemcpy(this->d_elem + dir*(su.Field_length())*(su.Total_length())*2 , su.D_elem(), su.Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); } template @@ -122,7 +122,7 @@ template void PLEGMA_Gauge::stoutSmearing(PLEGMA_Gauge &uin, int nSmear, double rho, int D3D4, bool S4D){ if(nSmear < 1){ cudaMemcpy(this->D_elem(), uin.D_elem(), this->Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); return; } int smearD = (S4D)? 3:D3D4; @@ -158,7 +158,7 @@ void PLEGMA_Gauge::stoutSmearing(PLEGMA_Gauge &uin, int nSmear, do if(D3D4 == 3){// || S4D int offset = 3*(tmp1.Field_length())*(tmp1.Total_length())*2; cudaMemcpy(this->D_elem() + offset, uin.D_elem() + offset, tmp1.Bytes_total(), cudaMemcpyDeviceToDevice ); - checkCudaError(); + checkQudaError(); } for(int idir = 0; idir < D3D4 ; idir++){ delete u_s1[idir]; @@ -186,7 +186,7 @@ template void PLEGMA_Gauge::APEsmearing(PLEGMA_Gauge &uin, int nSmear, double alpha, int D3D4){ if(nSmear < 1){ cudaMemcpy(this->D_elem(), uin.D_elem(), this->Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); return; } PLEGMA_Su3field tmp1(BOTH); @@ -220,7 +220,7 @@ void PLEGMA_Gauge::APEsmearing(PLEGMA_Gauge &uin, int nSmear, doub if(D3D4 == 3){ int offset = 3*(tmp1.Field_length())*(tmp1.Total_length())*2; cudaMemcpy(this->D_elem() + offset, uin.D_elem() + offset, tmp1.Bytes_total(), cudaMemcpyDeviceToDevice ); - checkCudaError(); + checkQudaError(); } for(int idir = 0; idir < D3D4 ; idir++){ diff --git a/lib/PLEGMA_Propagator.cu b/lib/PLEGMA_Propagator.cu index da408ef5..9d1a5231 100644 --- a/lib/PLEGMA_Propagator.cu +++ b/lib/PLEGMA_Propagator.cu @@ -26,7 +26,7 @@ absorbVectorToHost(PLEGMA_Vector &vec, int nu, int c2){ pointVec_dev = vec.D_elem() + mu*N_COLS*HGC_localVolume*2 + c1*HGC_localVolume*2; cudaMemcpy(pointProp_host,pointVec_dev,HGC_localVolume*2*sizeof(Float),cudaMemcpyDeviceToHost); } - checkCudaError(); + checkQudaError(); } // Prop4D <- Vec4D @@ -45,7 +45,7 @@ void PLEGMA_Propagator::absorb(PLEGMA_Vector &vec, int nu, int c2) cudaMemcpy(pointProp_dev,pointVec_dev,HGC_localVolume*2*sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } // Prop4D <- Vec4D (it) @@ -68,7 +68,7 @@ void PLEGMA_Propagator::absorb(PLEGMA_Vector &vec, int global_it, } } comm_barrier(); - checkCudaError(); + checkQudaError(); } //Prop4D <- Vec3D @@ -91,7 +91,7 @@ void PLEGMA_Propagator::absorb(PLEGMA_Vector3D &vec, int global_it } } comm_barrier(); - checkCudaError(); + checkQudaError(); } template @@ -220,7 +220,7 @@ absorbTimeSliceFromHost(PLEGMA_Propagator &prop, cudaMemcpy(this->d_elem,this->h_elem, N_SPINS*N_SPINS*N_COLS*N_COLS*V3*2*sizeof(Float), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); } //Prop3D <- Vec4D @@ -244,7 +244,7 @@ void PLEGMA_Propagator3D::absorb(PLEGMA_Vector &vec, int global_it else cudaMemset(pointer_dst, 0, V3*2 * sizeof(Float)); } - checkCudaError(); + checkQudaError(); } //Prop3D <- Vec3D @@ -260,7 +260,7 @@ void PLEGMA_Propagator3D::absorb(PLEGMA_Vector3D &vec, int nu, int pointer_src = (vec.D_elem() + mu*N_COLS*V3*2 + c1*V3*2); cudaMemcpy(pointer_dst, pointer_src, V3*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } template class PLEGMA_Propagator; diff --git a/lib/PLEGMA_QLoops.cu b/lib/PLEGMA_QLoops.cu index b57596ee..6c61266f 100644 --- a/lib/PLEGMA_QLoops.cu +++ b/lib/PLEGMA_QLoops.cu @@ -89,7 +89,7 @@ void PLEGMA_QLoops::contractG5(PLEGMA_Vector &x_l, PLEGMA_Vector(x_l); auto vtex_r = toTexture(x_r); contractG5_bilinear(toField2(*this), *vtex_l, *vtex_r, accum_sign); - checkCudaError(); + checkQudaError(); } template diff --git a/lib/PLEGMA_Random.cu b/lib/PLEGMA_Random.cu index 92481695..408c3517 100644 --- a/lib/PLEGMA_Random.cu +++ b/lib/PLEGMA_Random.cu @@ -77,14 +77,14 @@ PLEGMA_RNG::~PLEGMA_RNG(){ PLEGMA_printf("Free array of random numbers with rng_size: %.2f MB\n", ((float)rng_size * (float)sizeof(cuRNGState))/(1024*1024)); rng_size = 0; state = NULL; - checkCudaError(); + checkQudaError(); } /*! @brief Generating random numbers from random distribution */ /*! @brief Restore CURAND array states initialization */ void PLEGMA_RNG::restore() { cudaMemcpy(state, backup_state, rng_size * sizeof(cuRNGState), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); hostFree(backup_state, rng_size * sizeof(cuRNGState)); } @@ -92,6 +92,6 @@ void PLEGMA_RNG::restore() { void PLEGMA_RNG::backup() { hostMalloc(backup_state, rng_size * sizeof(cuRNGState)); cudaMemcpy(backup_state, state, rng_size * sizeof(cuRNGState), cudaMemcpyDeviceToHost); - checkCudaError(); + checkQudaError(); } diff --git a/lib/PLEGMA_Su3field.cu b/lib/PLEGMA_Su3field.cu index 4e004fca..c853ada1 100644 --- a/lib/PLEGMA_Su3field.cu +++ b/lib/PLEGMA_Su3field.cu @@ -17,7 +17,7 @@ template void PLEGMA_Su3field::absorbDir_device(PLEGMA_Gauge &u,int dir){ cudaMemcpy(this->d_elem, u.D_elem()+dir*(this->field_length)*(this->total_length)*2, this->Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); } template @@ -133,7 +133,7 @@ void PLEGMA_Su3field::staples(PLEGMA_Su3field **u, int dir, PLEGMA } tmp1.shift(*this,4+dir); cudaMemcpy(this->D_elem(), tmp1.D_elem(), this->Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); } @@ -147,7 +147,7 @@ void PLEGMA_Su3field::wilsonLineUpdate(PLEGMA_Su3field &inOut, PLE */ if( !((dirOr >= 0) && (dirOr <= 7)) ) PLEGMA_error("Error you provided a direction which is not supported"); cudaMemcpy(tmp.D_elem(), inOut.D_elem(), tmp.Bytes_total(), cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); if(dirOr > 3){ // x->->->-> if(reverse) this->Udag(); diff --git a/lib/PLEGMA_Vector.cu b/lib/PLEGMA_Vector.cu index 01e8013f..6293f96c 100644 --- a/lib/PLEGMA_Vector.cu +++ b/lib/PLEGMA_Vector.cu @@ -71,7 +71,7 @@ void PLEGMA_Vector::gaussianSmearing(PLEGMA_Vector &vecIn, cudaMemcpy(this->D_elem(),vecIn.D_elem(), this->Bytes_total(),cudaMemcpyDeviceToDevice); - checkCudaError(); + checkQudaError(); if(vecIn.IsAllocHost()) { vecIn.load(); // restoring vecIn @@ -141,7 +141,7 @@ void PLEGMA_Vector::absorb(PLEGMA_Propagator3D &prop, int global_i } } comm_barrier(); - checkCudaError(); + checkQudaError(); } // vec4D <- prop4D (it) @@ -164,7 +164,7 @@ void PLEGMA_Vector::absorb(PLEGMA_Propagator &prop, int global_it, } } comm_barrier(); - checkCudaError(); + checkQudaError(); } // vec4D <- prop4D @@ -179,7 +179,7 @@ void PLEGMA_Vector::absorb(PLEGMA_Propagator &prop, int nu , int c pointer_src = (prop.D_elem() + mu*N_SPINS*N_COLS*N_COLS*V4*2 + nu*N_COLS*N_COLS*V4*2 + c1*N_COLS*V4*2 + c2*V4*2); cudaMemcpy(pointer_dst, pointer_src, V4*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } template @@ -194,7 +194,7 @@ void PLEGMA_Vector::dilutespin(PLEGMA_Vector &vecIn, int spin){ cudaMemcpy((this->d_elem + ((c1 + mu*N_COLS)*HGC_localVolume)*2), pointer_src, HGC_localVolume*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } } - checkCudaError(); + checkQudaError(); } template @@ -209,7 +209,7 @@ void PLEGMA_Vector::dilutecolor(PLEGMA_Vector &vecIn, int color){ cudaMemcpy((this->d_elem + ((c1 + mu*N_COLS)*HGC_localVolume)*2), pointer_src, HGC_localVolume*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } } - checkCudaError(); + checkQudaError(); } template @@ -225,7 +225,7 @@ void PLEGMA_Vector::dilutespincolor(PLEGMA_Vector &vecIn, int spin cudaMemcpy((this->d_elem + ((c1 + mu*N_COLS)*HGC_localVolume)*2), pointer_src, HGC_localVolume*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } } - checkCudaError(); + checkQudaError(); } template @@ -238,7 +238,7 @@ void PLEGMA_Vector::diluteSpinDisplace(PLEGMA_Vector &vecIn, int s cudaMemcpy((this->d_elem + ((c1 + spin1*N_COLS)*HGC_localVolume)*2), pointer_src, HGC_localVolume*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } @@ -392,7 +392,7 @@ namespace plegma{ pointer_src = (prop.D_elem() + mu*N_SPINS*N_COLS*N_COLS*V3*2 + nu*N_COLS*N_COLS*V3*2 + c1*N_COLS*V3*2 + c2*V3*2); cudaMemcpy(pointer_dst, pointer_src, V3*2 * sizeof(Float), cudaMemcpyDeviceToDevice); } - checkCudaError(); + checkQudaError(); } // vec3D <- Prop4D @@ -433,7 +433,7 @@ namespace plegma{ } } - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_FT.cuh b/lib/kernels/PLEGMA_FT.cuh index 53e5e1b1..80f01a90 100644 --- a/lib/kernels/PLEGMA_FT.cuh +++ b/lib/kernels/PLEGMA_FT.cuh @@ -56,7 +56,7 @@ static void FT_dot(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vect Float2 *x; cudaMalloc((void**)&x, V*2*sizeof(Float)); cudaMemset(x,0,V*2*sizeof(Float)); - checkCudaError(); + checkQudaError(); for(int imom = 0; imom < Nmom; imom++){ createMomField(x,mom[imom],ft.Dims(),-sign); // change sign to compensate dagger for(int idf = 0 ; idf < f.Field_length(); idf++) @@ -82,7 +82,7 @@ static void FT_gemv(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vec cudaMalloc((void**)&x, V*2*sizeof(Float)); cudaMemset(x,0,V*2*sizeof(Float)); cudaMalloc((void**)&d_res, f.Field_length() * ft.DimT() * 2*sizeof(Float)); - checkCudaError(); + checkQudaError(); Float2 h_res[f.Field_length()*ft.DimT()]; Float2 *h_ft = (Float2 *) ft.H_elem(); Float one[2] = {1.,0.}, zero[2] = {0.,0.}; @@ -131,12 +131,12 @@ static void fourier_transform_3D_k(PLEGMA_FT &ft, const PLEGMA_Field, d_partial_block, toField2(field), texMomList, it, sign); size_t alloc_size=ft.Nmoms()*site_size*ps.tp.grid.x*2*sizeof(Float); cudaMalloc((void**)&d_partial_block, alloc_size); - checkCudaError(); + checkQudaError(); run(ps, "fourier_transform_3D_kernel", fourier_transform_3D_kernel, d_partial_block, toField2(field), texMomList, it, sign); Float *h_partial_block = NULL; hostMalloc(h_partial_block,alloc_size); cudaMemcpy(h_partial_block,d_partial_block,alloc_size,cudaMemcpyDeviceToHost); - checkCudaError(); + checkQudaError(); cudaFree(d_partial_block); int gridDimX = ps.tp.grid.x; Float *reduction; diff --git a/lib/kernels/PLEGMA_SU3_projection.cuh b/lib/kernels/PLEGMA_SU3_projection.cuh index 03aea26c..30020c3f 100644 --- a/lib/kernels/PLEGMA_SU3_projection.cuh +++ b/lib/kernels/PLEGMA_SU3_projection.cuh @@ -80,5 +80,5 @@ static void su3Projection_k(PLEGMA_Su3field &S){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (S.Total_length() + blockDim.x -1)/blockDim.x , 1 , 1); su3Projection_kernel<<>>(toField2(S)); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_WFlow.cuh b/lib/kernels/PLEGMA_WFlow.cuh index aa168ef2..6a015f43 100644 --- a/lib/kernels/PLEGMA_WFlow.cuh +++ b/lib/kernels/PLEGMA_WFlow.cuh @@ -112,7 +112,7 @@ __inline__ void GFlow_substep( gauge2 W, gauge2 Z, FloatE e_work cudaDeviceSynchronize(); WUpdate<<>>( W, Z ); - checkCudaError(); + checkQudaError(); } @@ -142,7 +142,7 @@ __inline__ void unitarize_dev( gauge2 gauge ) dim3 gridDim( (gauge.volume() + blockDim.x -1)/blockDim.x, 1, 1); unitarize_dev_kernel<<>>( gauge ); - checkCudaError(); + checkQudaError(); } //##################################################################################### @@ -198,7 +198,7 @@ static FloatG calcPlaqStaplesDef(gauge2 gaugep){ cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDim.x * sizeof(FloatG) , cudaMemcpyDeviceToHost); cudaFree(d_partial_plaq); - checkCudaError(); + checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++) plaquette += h_partial_plaq[i]; diff --git a/lib/kernels/PLEGMA_contractG5_bilinear.cuh b/lib/kernels/PLEGMA_contractG5_bilinear.cuh index 1de0f3d7..26037485 100644 --- a/lib/kernels/PLEGMA_contractG5_bilinear.cuh +++ b/lib/kernels/PLEGMA_contractG5_bilinear.cuh @@ -34,5 +34,5 @@ template static void contractG5_bilinear( generic2 qLoops, vectorTex& v_l, vectorTex& v_r, Float accum_sign){ ProfileStruct ps(qLoops.volume()); run(ps, "contractG5_bilinear_kernel", contractG5_bilinear_kernel, qLoops, v_l, v_r, accum_sign); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_covD.cuh b/lib/kernels/PLEGMA_covD.cuh index f3007939..0f776272 100644 --- a/lib/kernels/PLEGMA_covD.cuh +++ b/lib/kernels/PLEGMA_covD.cuh @@ -32,5 +32,5 @@ template static void covD_k(vector2 out, vectorTex v, gaugeTex g, int dirOr){ ProfileStruct ps(out.volume()); tuneAndRun(ps, "covD_kernel", covD_kernel, out, v, g, dirOr); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_field_utils.cuh b/lib/kernels/PLEGMA_field_utils.cuh index 604d4802..f3c8f4ce 100644 --- a/lib/kernels/PLEGMA_field_utils.cuh +++ b/lib/kernels/PLEGMA_field_utils.cuh @@ -25,7 +25,7 @@ template static void cudaCast(pFloat2 out, pFloat2 in){ ProfileStruct ps(out.volume()); // here we can actually use any size tuneAndRun(ps, "cast_kernel_size_"+std::to_string(out.site_size), cast_kernel, out, in); - checkCudaError(); + checkQudaError(); } @@ -142,7 +142,7 @@ void conjugate_k(PLEGMA_Field& inOut){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (field.volume() + blockDim.x -1)/blockDim.x , 1 , 1); conjugate_kernel<<>>(field); - checkCudaError(); + checkQudaError(); } @@ -325,7 +325,7 @@ static void traceMulFmunuSu3FmunuSu3_k(PLEGMA_Field &F, PLEGMA_Fmunu, F.D_elem(), RA, toField2(B), RC, toField2(D)); - checkCudaError(); + checkQudaError(); } template @@ -353,5 +353,5 @@ static void trPmunu_k(PLEGMA_Field &f,PLEGMA_Gauge &gauge, std:: ProfileStruct ps(gauge.Total_length()); if(std::get<0>(munu) == std::get<1>(munu)) PLEGMA_error("For Pmunu cannot have mu == nu"); tuneAndRun(ps,"trPmunu_kernel",trPmunu_kernel,f.D_elem(),toField2(gauge),std::get<0>(munu),std::get<1>(munu)); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_fmunu_utils.cuh b/lib/kernels/PLEGMA_fmunu_utils.cuh index 7c32b983..54e82753 100644 --- a/lib/kernels/PLEGMA_fmunu_utils.cuh +++ b/lib/kernels/PLEGMA_fmunu_utils.cuh @@ -96,5 +96,5 @@ static void clover_leaves_k(PLEGMA_Fmunu &fmunu, PLEGMA_Gauge &gau assert(fmunu.checkVolume(gauge)); ProfileStruct ps(gauge.Total_length()); tuneAndRun(ps,"clover_leaves_kernel", clover_leaves_kernel, toField2(fmunu), toField2(gauge)); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_gFixing.cuh b/lib/kernels/PLEGMA_gFixing.cuh index 70d784e3..73e4d9ea 100644 --- a/lib/kernels/PLEGMA_gFixing.cuh +++ b/lib/kernels/PLEGMA_gFixing.cuh @@ -111,10 +111,10 @@ static void gFixingLandau_k(PLEGMA_Gauge &u_gFixed, PLEGMA_Gauge & for(int eo=0; eo < 2; eo++){ u_gFixed.communicateGhost(-1, DIR_MINUS); gTransformLandau_kernel<<>>(toField2(g), toField2(u_gFixed), rnd.D_elem(),overelaxPar,eo); - checkCudaError(); + checkQudaError(); g.communicateGhost(-1, DIR_PLUS); gTransformMulALandau_kernel<<>>(toField2(g), toField2(u_gFixed),eo); - checkCudaError(); + checkQudaError(); rnd.random(Uniform); } diff --git a/lib/kernels/PLEGMA_gammas_scatt.cuh b/lib/kernels/PLEGMA_gammas_scatt.cuh index 62d789f1..4872c77e 100644 --- a/lib/kernels/PLEGMA_gammas_scatt.cuh +++ b/lib/kernels/PLEGMA_gammas_scatt.cuh @@ -39,7 +39,7 @@ namespace plegma{ apply_gamma_scatt_vector_kernel<<>>(inOut, r); break; } - checkCudaError(); + checkQudaError(); } } diff --git a/lib/kernels/PLEGMA_gauge_utils.cuh b/lib/kernels/PLEGMA_gauge_utils.cuh index 08c42f05..02f23af7 100644 --- a/lib/kernels/PLEGMA_gauge_utils.cuh +++ b/lib/kernels/PLEGMA_gauge_utils.cuh @@ -25,7 +25,7 @@ static void U3xU1_k( PLEGMA_Gauge &u3Out, PLEGMA_Gauge &u3In, ProfileStruct ps(u3Out.Total_length()); tuneAndRun(ps, "U3xU1_kernel", U3xU1_kernel, toField2(u3Out), toField2(u3In), toField2(u1)); - checkCudaError(); + checkQudaError(); } @@ -55,5 +55,5 @@ static void scale_dir_wise(gauge2 gauge, Float* scale){ cudaMemcpy( d_scale, scale, N_DIMS*sizeof(Float2),cudaMemcpyHostToDevice); scale_dir_wise_kernel<<>>(gauge, d_scale); cudaFree(d_scale); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index 7c9ba868..e99f0a6f 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -264,8 +264,8 @@ void PLEGMA_kernel_tuner::apply(const qudaStream_t &stream){ #endif } -template -void PLEGMA_kernel_tuner::apply(){ apply(0); } +//template +//void PLEGMA_kernel_tuner::apply(){ apply(0); } template void PLEGMA_kernel_tuner::run(){ @@ -276,7 +276,7 @@ void PLEGMA_kernel_tuner::run(){ if(!ps.tuned) ps.tp = tuneLaunch(*this, QUDA_TUNE_NO, (QudaVerbosity) HGC_verbosity); launchKernel(ps.tp,0); #endif - checkCudaError(); + checkQudaError(); } template diff --git a/lib/kernels/PLEGMA_plaquette.cuh b/lib/kernels/PLEGMA_plaquette.cuh index b1af94c4..307e71dd 100644 --- a/lib/kernels/PLEGMA_plaquette.cuh +++ b/lib/kernels/PLEGMA_plaquette.cuh @@ -92,7 +92,7 @@ static void calculatePlaquette_host(ProfileStruct& ps, TG gTex, Float& plaquette if(h_partial_plaq == NULL) PLEGMA_error("Error allocate memory for host partial plaq"); cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , cudaMemcpyDeviceToHost); cudaFree(d_partial_plaq); - checkCudaError(); + checkQudaError(); plaquette = 0.; for(int i = 0 ; i < gridDimX ; i++) diff --git a/lib/kernels/PLEGMA_plaquetteCorners.cuh b/lib/kernels/PLEGMA_plaquetteCorners.cuh index d5a07e97..47537c49 100644 --- a/lib/kernels/PLEGMA_plaquetteCorners.cuh +++ b/lib/kernels/PLEGMA_plaquetteCorners.cuh @@ -109,7 +109,7 @@ static void calculatePlaquetteCorners_host(ProfileStruct& ps, gaugeTex g if(h_partial_plaq == NULL) PLEGMA_error("Error allocate memory for host partial plaq"); cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , cudaMemcpyDeviceToHost); cudaFree(d_partial_plaq); - checkCudaError(); + checkQudaError(); plaquette = 0.; for(int i = 0 ; i < gridDimX ; i++) diff --git a/lib/kernels/PLEGMA_propagator_utils.cuh b/lib/kernels/PLEGMA_propagator_utils.cuh index 9e2f6611..b6646f37 100644 --- a/lib/kernels/PLEGMA_propagator_utils.cuh +++ b/lib/kernels/PLEGMA_propagator_utils.cuh @@ -25,7 +25,7 @@ static void apply_gamma_prop(LEFTRIGHT LR, PLEGMA_Propagator& InOut, GAMM apply_gamma_prop_kernel<<>>(prop, r); break; } - checkCudaError(); + checkQudaError(); } @@ -83,7 +83,7 @@ void apply_boundaries(Float *inOut, int t0){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (HGC_localVolume + blockDim.x -1)/blockDim.x , 1 , 1); apply_boundaries_kernel<<>>(inOut,t0); - checkCudaError(); + checkQudaError(); } @@ -130,5 +130,5 @@ void rotateToPhysicalBase(Float* inOut, int sign){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (HGC_localVolume + blockDim.x -1)/blockDim.x , 1 , 1); rotateToPhysicalBase_kernel<<>>((Float*) inOut,sign); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_scattreductions.cuh b/lib/kernels/PLEGMA_scattreductions.cuh index 5ea84a1c..3ffd25b3 100644 --- a/lib/kernels/PLEGMA_scattreductions.cuh +++ b/lib/kernels/PLEGMA_scattreductions.cuh @@ -50,12 +50,14 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); +// cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { + auto error= qudaGetLastError(); + if(error != QUDA_SUCCESS) { + errorQuda("Failed to clear error state %s\n", qudaGetLastErrorString().c_str()); PLEGMA_printf("Error in allocating d_partial_block %d\n",alloc_size); - cudaFree(d_partial_block); + device_free(d_partial_block); return; } @@ -64,15 +66,33 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator listGammas.size = gammas.size(); if (gammas.size()==0){ - cudaMalloc((void**)&listGammas.array, sizeof(GAMMAS_SCATT)); - checkCudaError(); + listGammas.array=device_malloc(sizeof(GAMMAS_SCATT)); + checkQudaError(); + } + +// cudaMalloc((void**)&listGammas.array, sizeof(GAMMAS_SCATT)); +// checkCudaError(); } else{ - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS_SCATT)); - checkCudaError(); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); - checkCudaError(); + listGammas.array=device_malloc(gammas.size()*sizeof(GAMMAS_SCATT)); +// cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS_SCATT)); + auto error= qudaGetLastError(); + if(error != QUDA_SUCCESS) { + errorQuda("Failed to clear error state %s\n", qudaGetLastErrorString().c_str()); + PLEGMA_printf("Error in allocating listGammas.array %d\n",gammas.size()*sizeof(GAMMAS_SCATT)); + device_free(d_partial_block); + return; + } + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS_SCATT), qudaMemcpyHostToDevice); + auto error= qudaGetLastError(); + if(error != QUDA_SUCCESS) { + errorQuda("Failed to copy from host to device %s\n", qudaGetLastErrorString().c_str()); + PLEGMA_printf("Error in allocating %d\n",gammas.size()*sizeof(GAMMAS_SCATT)); + return; + } + +// cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); } //loop over the bunches of timeslices passed to device @@ -353,8 +373,10 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + + d_partial_block=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__,alloc_size*sizeof(Float2)); + +// cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. cudaError_t error=cudaPeekAtLastError(); @@ -368,12 +390,18 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator KernelArr listGammas_i, listGammas_f; listGammas_i.size = gammas_i.size(); listGammas_f.size = gammas_f.size(); - cudaMalloc((void**)&listGammas_i.array, gammas_i.size()*sizeof(GAMMAS_SCATT)); - cudaMalloc((void**)&listGammas_f.array, gammas_f.size()*sizeof(GAMMAS_SCATT)); - checkCudaError(); + + listGammas_i.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_i.size()*sizeof(GAMMAS_SCATT)); + + listGammas_f.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_f.size()*sizeof(GAMMAS_SCATT)); + + +// cudaMalloc((void**)&listGammas_i.array, gammas_i.size()*sizeof(GAMMAS_SCATT)); +// cudaMalloc((void**)&listGammas_f.array, gammas_f.size()*sizeof(GAMMAS_SCATT)); + checkQudaError(); cudaMemcpy(listGammas_i.array, gammas_i.data(), gammas_i.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); cudaMemcpy(listGammas_f.array, gammas_f.data(), gammas_f.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); if(HGC_verbosity > 2){ PLEGMA_printf("site_size= %d\n", listGammas_f.size*listGammas_i.size*N_SPINS*N_SPINS); } diff --git a/lib/kernels/PLEGMA_scattreductionsPiPi.cuh b/lib/kernels/PLEGMA_scattreductionsPiPi.cuh index e6ab72c0..82ff329a 100644 --- a/lib/kernels/PLEGMA_scattreductionsPiPi.cuh +++ b/lib/kernels/PLEGMA_scattreductionsPiPi.cuh @@ -102,9 +102,9 @@ static void PhixGxPhi_host( ProfileStruct &ps, PLEGMA_ScattCorrelator KernelArr listGammas; listGammas.size = gammas.size(); cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS_SCATT)); - checkCudaError(); + checkQudaError(); cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); if(HGC_verbosity > 2) PLEGMA_printf("site_size= %d\n", listGammas.size*N_SPINS*N_COLS); diff --git a/lib/kernels/PLEGMA_seqSourceNucleon.cu b/lib/kernels/PLEGMA_seqSourceNucleon.cu index a582bcd0..a47e7ec4 100644 --- a/lib/kernels/PLEGMA_seqSourceNucleon.cu +++ b/lib/kernels/PLEGMA_seqSourceNucleon.cu @@ -141,7 +141,7 @@ template static void contractNucleonSeqSource(vector2 &vec, propTex& prop1, propTex& prop2, WHICHPROJECTOR proj, WHICHPARTICLE particle, bool isTwoPropDiff, int c_nu, int c_c2){ ProfileStruct ps(vec.volume()); tuneAndRun(ps,"contractNucleonSeqSource", contractNucleonSeqSource_kernel, vec, prop1, prop2, proj, particle, isTwoPropDiff, c_nu, c_c2); - checkCudaError(); + checkQudaError(); } template diff --git a/lib/kernels/PLEGMA_shifts.cuh b/lib/kernels/PLEGMA_shifts.cuh index 90f766fe..1b98401f 100644 --- a/lib/kernels/PLEGMA_shifts.cuh +++ b/lib/kernels/PLEGMA_shifts.cuh @@ -20,7 +20,7 @@ static void shiftField(PLEGMA_Field &Fin, PLEGMA_Field &Fout, int assert(Fin.checkVolume(Fout)); ProfileStruct ps( Fin.Total_length() ); tuneAndRun(ps, "shifts1_kernel", shifts1_kernel, toField2(Fin), toField2(Fout),dirOr); - checkCudaError(); + checkQudaError(); } @@ -45,7 +45,7 @@ static void shiftField(PLEGMA_Field &Fin, PLEGMA_Field &Fout, int assert(Fin.checkVolume(Fout)); ProfileStruct ps( Fin.Total_length() ); tuneAndRun(ps, "shifts2_kernel", shifts2_kernel, toField2(Fin), toField2(Fout),dirOr1,dirOr2); - checkCudaError(); + checkQudaError(); } @@ -74,5 +74,5 @@ static void shiftField(PLEGMA_Field &Fin, PLEGMA_Field &Fout, int assert(Fin.checkVolume(Fout)); ProfileStruct ps( Fin.Total_length() ); tuneAndRun(ps, "shifts3_kernel", shifts3_kernel, toField2(Fin), toField2(Fout),dirOr1,dirOr2,dirOr3); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_su3field.cuh b/lib/kernels/PLEGMA_su3field.cuh index 0910d27a..e0ae4963 100644 --- a/lib/kernels/PLEGMA_su3field.cuh +++ b/lib/kernels/PLEGMA_su3field.cuh @@ -124,7 +124,7 @@ static void U_plus_eq_aU_k( PLEGMA_Su3field &A, PLEGMA_Su3field assert(A.checkVolume(B)); ProfileStruct ps(A.Total_length()); run(ps, "U_plus_eq_aU_kernel", U_plus_eq_aU_kernel, toField2(A), toField2(B), c); - checkCudaError(); + checkQudaError(); } template @@ -132,7 +132,7 @@ static void traceHerExpMap_k(PLEGMA_Su3field &A, PLEGMA_Su3field assert(A.checkVolume(B)); ProfileStruct ps(A.Total_length()); tuneAndRun(ps, "traceHerExpMap_kernel", traceHerExpMap_kernel, toField2(A), toField2(B)); - checkCudaError(); + checkQudaError(); } template @@ -140,14 +140,14 @@ static void Udag_k(PLEGMA_Su3field &A, PLEGMA_Su3field &B){ assert(A.checkVolume(B)); ProfileStruct ps(A.Total_length()); tuneAndRun(ps, "Udag_kernel", Udag_kernel,toField2(A), toField2(B)); - checkCudaError(); + checkQudaError(); } template static void Udag_k(PLEGMA_Su3field &A){ ProfileStruct ps(A.Total_length()); run(ps, "Udag_kernel", Udag_kernel,toField2(A)); - checkCudaError(); + checkQudaError(); } template @@ -155,7 +155,7 @@ static void UxU_k(PLEGMA_Su3field &A, PLEGMA_Su3field &B, PLEGMA assert(A.checkVolume(B,C)); ProfileStruct ps(A.Total_length()); tuneAndRun(ps, "UxU_kernel", UxU_kernel,toField2(A), toField2(B),toField2(C)); - checkCudaError(); + checkQudaError(); } template @@ -163,7 +163,7 @@ static void UxUdag_k(PLEGMA_Su3field &A, PLEGMA_Su3field &B, PLE assert(A.checkVolume(B,C)); ProfileStruct ps(A.Total_length()); tuneAndRun(ps, "UxUdag_kernel", UxUdag_kernel, toField2(A), toField2(B),toField2(C)); - checkCudaError(); + checkQudaError(); } template @@ -174,13 +174,14 @@ static void sum_real_trace_host(ProfileStruct& ps, PLEGMA_Su3field &su3M int gridDimX = ps.tp.grid.x; hostMalloc(h_partial_sum, gridDimX * sizeof(Float) ); - cudaMalloc((void**)&d_partial_sum, gridDimX * sizeof(Float)); +// cudaMalloc((void**)&d_partial_sum, gridDimX * sizeof(Float)); + d_partial_sum=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gridDimX * sizeof(Float)); sum_real_trace_kernel<<>>(toField2(su3M), d_partial_sum); cudaMemcpy(h_partial_sum, d_partial_sum , gridDimX * sizeof(Float) , cudaMemcpyDeviceToHost); cudaFree(d_partial_sum); - checkCudaError(); + checkQudaError(); for(int i = 0 ; i < gridDimX ; i++) sum += h_partial_sum[i]; diff --git a/lib/kernels/PLEGMA_threep_local_stochastic.cu b/lib/kernels/PLEGMA_threep_local_stochastic.cu new file mode 100644 index 00000000..eced1caf --- /dev/null +++ b/lib/kernels/PLEGMA_threep_local_stochastic.cu @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include + +using namespace plegma; +template +struct KernelArr {T* array; int size;}; + +template +__global__ void threep_local_stochastic_device(Float2* block2, + vectorTex prop1Tex, vectorTex prop2Tex, + KernelArr listGammas, + int it, int time_step, int maxT, int4 source, + int signProps, bool runFT, tex_mom_list moms){ + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + Float2 accum[N_SPINS*N_SPINS]; // max value of gammas + #pragma unroll + for(int i = 0; i < N_SPINS*N_SPINS; i++) + accum[i]=0; + + if (sid3D < DGC_localVolume3D){ + /*Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; + Float2 R[N_SPINS][N_SPINS]; + prop1Tex.get(prop1,vid); + prop2Tex.get(prop2,vid); + partial_trace_mul_Prop_Prop(R,prop1,prop2); + + for(int iop = 0; iop < listGammas.size; iop++){ + int opId=listGammas.array[iop]; + if (signProps >0){ + accum[iop]=trace_gamma_S(opId,TMP,R); + } + else if (signProps<0){ + accum[iop]=trace_gamma_S(opId,TMM,R); + } + else{ + accum[iop]=trace_gamma_S(opId,NOROT,R); + } + }*/ + } + + int source_pos[3] = {source.x, source.y, source.z}; + + if(runFT){ + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + fourier_transform_3D(block2, accum, shared_cache, listGammas.size, sid3D, source_pos, moms, 0, +1, time_step, tid); + } else{ + if (sid3D < DGC_localVolume3D) + for(int iop = 0; iop < listGammas.size; iop++) + block2[(tid*DGC_localVolume3D + sid3D)*listGammas.size +iop] = accum[iop]; + } +} + +template +static void threep_local_stochastic_host(ProfileStruct &ps, Float2 *result, + PLEGMA_Correlator &corr, + PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, + int signProps, std::vector& gammas ){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); + + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + size_t volume = corr.getVolSize()/t_size; + + + size_t size = corr.getTotalSize()/t_size*time_step; + int site_size = corr.getSiteSize(); + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + + KernelArr listGammas; + listGammas.size = gammas.size(); + cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); + cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("time_step = %d, t_size = %d, maxT = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", time_step, t_size, maxT, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto propTex1 = toTexture(prop1); +// auto propTex2 = toTexture(prop2); + + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess || h_partial_block==NULL) goto exit; + for(int it=0; it < t_size; it+=time_step) { + int t_step = std::min(t_size-it, time_step); + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*t_step; +/* threep_local_stochastic_device + <<>> + (d_partial_block, *propTex1, *propTex2, listGammas, it, t_step, maxT, source, signProps, runFT, *moms);*/ + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + if(runFT==true){ + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) { + result[(it*volume+v)*site_size+i] = 0; + for(int j = 0 ; j < accumX; j++) + result[(it*volume+v)*site_size+i] += + h_partial_block[(v*site_size+i)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) + result[(it*volume+v)*site_size+i] += + h_partial_block[v*site_size+i]; + } + } + + exit: + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); + cudaFree(listGammas.array); +} + +template +void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas) { +#ifdef PLEGMA_NUCLEON_3PF_FIX_SINK + if(gammas.size() <= 0) + PLEGMA_error("Error the container of gamma matrices cannot be zero"); + if(gammas.size() > N_SPINS*N_SPINS) + PLEGMA_error("Error maximum number of gamma matrices is 16"); + + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + int site_size = gammas.size(); + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + site_size = gammas.size(); + ProfileStruct ps(HGC_localVolume3D, (runFT==true) ? site_size*sizeof(Float2) : 0); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + tune( ps, "threep_local_stochastic", threep_local_stochastic_host, + ps, result, corr, prop1, prop2, signProps, gammas); + run( ps, "threep_local_stochastic", threep_local_stochastic_host, + ps, result, corr, prop1, prop2, signProps, gammas); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(corr.H_elem()), + MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +#else + PLEGMA_error("You must enable PLEGMA_NUCLEON_3PF_FIX_SINK\n"); +#endif +} + +template void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas); +template void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas); diff --git a/lib/kernels/PLEGMA_threep_twoD_stochastic.cu b/lib/kernels/PLEGMA_threep_twoD_stochastic.cu new file mode 100644 index 00000000..1a98a17e --- /dev/null +++ b/lib/kernels/PLEGMA_threep_twoD_stochastic.cu @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include + +using namespace plegma; +template +struct KernelArr {T* array; int size;}; + +template +__global__ void threep_twoD_device(Float2* block2, + vectorTex vect1Tex, vectorTex vect2Tex, + gaugeTex gaugeTex, KernelArr listGammas, + int it, int time_step, int maxT, int4 source, + int signProps, bool runFT, tex_mom_list moms, + int dir1, int dir2){ + int grid3D = gridDim.x/time_step; + int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; + int tid = blockIdx.x/grid3D; + // this takes into account the case where the source is in the local lattice + // and we need to start from it when we go over maxT + int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; + int vid = sid3D + t*DGC_localVolume3D; + + Float2 accum[N_SPINS*N_SPINS]; + #pragma unroll + for(int i = 0; i < N_SPINS*N_SPINS; i++) + accum[i]=0; + + if (sid3D < DGC_localVolume3D){ + Float2 vect1[N_SPINS][N_COLS]; + Float2 vect2[N_SPINS][N_COLS]; + Float2 R[N_SPINS][N_SPINS]; + Float2 su3_1[N_COLS][N_COLS]; + Float2 su3_2[N_COLS][N_COLS]; + + // BEGIN REGION + // This has been generated by python/generate_derivative.py + // The order has been optmized such that 18 data accesses have been commented out + // + term x, x, x+dir1, x+dir1+dir2 + vect1Tex.get(vect1,vid); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1,dir2); + partial_trace_mul_Vec_G1_G2_Vec(R,vect1,vect2,su3_1,su3_2); + + // - term x, x, x+dir1-dir2^, x+dir1-dir2 + /*vect1Tex.get(vect1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); vect2Tex.get(vect2,vid,dir1,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x+dir2, x+dir2, x+dir1^, x+dir1 + vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x-dir2, x-dir2, x+dir1-dir2, x+dir1 + vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*vect2Tex.get(vect2,vid,dir1);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x, x-dir1^, x-dir1, x-dir1+dir2 + vect1Tex.get(vect1,vid); gaugeTex.get(su3_1,dir1,vid,dir1); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x, x-dir1^, x-dir1-dir2^, x-dir1-dir2 + /*vect1Tex.get(vect1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); vect2Tex.get(vect2,vid,dir1,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x+dir2, x-dir1+dir2^, x-dir1^, x-dir1 + vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x-dir2, x-dir1-dir2^, x-dir1-dir2, x-dir1 + vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*vect2Tex.get(vect2,vid,dir1);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x+dir1, x^, x, x+dir2 + vect1Tex.get(vect1,vid,dir1); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid); vect2Tex.get(vect2,vid,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x+dir1, x^, x-dir2^, x-dir2 + /*vect1Tex.get(vect1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir2); vect2Tex.get(vect2,vid,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x-dir1, x-dir1, x-dir2^, x-dir2 + vect1Tex.get(vect1,vid,dir1); gaugeTex.get(su3_1,dir1,vid,dir1); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*vect2Tex.get(vect2,vid,dir2);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x-dir1, x-dir1, x, x+dir2 + /*vect1Tex.get(vect1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid); vect2Tex.get(vect2,vid,dir2); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x+dir1+dir2, x+dir2^, x^, x + vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ vect2Tex.get(vect2,vid); + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x-dir1+dir2, x-dir1+dir2, x^, x + vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ /*vect2Tex.get(vect2,vid);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // - term x+dir1-dir2, x-dir2^, x-dir2, x + vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir2); /*vect2Tex.get(vect2,vid);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // + term x-dir1-dir2, x-dir1-dir2, x-dir2, x + vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*vect2Tex.get(vect2,vid);*/ + partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); + + // END REGION + + for(int iop = 0; iop < listGammas.size; iop++){ + int opId=listGammas.array[iop]; + if(notZfac) accum[iop] = 0.0625*((signProps > 0) ? trace_gamma_S(opId,TMP,R) : trace_gamma_S(opId,TMM,R)); + else accum[iop] = trace_gamma_S(opId,NOROT,R); + } + } + + int site_size = listGammas.size; + int source_pos[3] = {source.x, source.y, source.z}; + if(runFT){ + extern __shared__ int ext_shared_cache[]; + Float2 *shared_cache = (Float2 *) ext_shared_cache; + fourier_transform_3D(block2, accum, shared_cache, site_size, sid3D, source_pos, moms, 0, +1, time_step, tid); + } else{ + if (sid3D < DGC_localVolume3D) + for(int iop = 0; iop < site_size; iop++) + block2[(tid*DGC_localVolume3D + sid3D)*site_size+iop] = accum[iop]; + } +} + +template +static void threep_twoD_host(ProfileStruct &ps, Float2 *result, PLEGMA_Correlator &corr, + PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, + int signProps, PLEGMA_Gauge& gauge, std::vector& gammas){ + + int t_size = corr.localT(); if(t_size==0) return; + int maxT = corr.endT() - corr.startT(); + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + size_t volume = corr.getVolSize()/t_size; + int extra=N_DIMS*(N_DIMS-1); + if(isZfac) extra*=N_SPINS*N_SPINS*N_COLS*N_COLS; + size_t size = corr.getTotalSize()/extra/t_size*time_step; + int site_size = corr.getSiteSize()/extra; + int4 source = corr.getSource(); + auto moms = corr.getTexMomList(); + + KernelArr listGammas; + listGammas.size = gammas.size(); + cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); + cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + + if(HGC_verbosity > 2) + if(corr.hasSource()) + printf("time_step = %d, t_size = %d, maxT = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", time_step, t_size, maxT, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + + size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; + + Float2 *h_partial_block = NULL; + Float2 *d_partial_block = NULL; + cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); + + auto vectorTex1 = toTexture(vect1); + auto vectorTex2 = toTexture(vect2); + auto gaugetex = toTexture(gauge); + + cudaError_t error=cudaPeekAtLastError(); + if(error != cudaSuccess || h_partial_block==NULL) goto exit; + for(int it=0; it < t_size; it+=time_step) { + for(int et=0; et < extra; et++) { + int dir1 = (et/(N_DIMS-1)) % N_DIMS; + int dir2 = et % (N_DIMS-1); + if(dir2>=dir1) dir2++; + if(isZfac) { + int tt = et/(N_DIMS*(N_DIMS-1)); + mu=tt/N_SPINS/N_COLS/N_COLS; + nu=(tt/N_COLS/N_COLS)%N_SPINS; + c1=(tt/N_COLS)%N_COLS; + c2=tt%N_COLS; + } + int t_step = std::min(t_size-it, time_step); + dim3 grid = ps.tp.grid; + grid.x = (grid.x/time_step)*t_step; + threep_twoD_device + <<>> + (d_partial_block, *vectorTex1, *vectorTex2, *gaugetex, listGammas, it, t_step, maxT, + source, signProps, runFT, *moms, dir1,dir2,mu,nu,c1,c2); + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), cudaMemcpyDeviceToHost); + error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + + if(runFT==true){ + int accumX = ps.tp.grid.x/time_step; + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) { + result[((it*volume+v)*extra+et)*site_size+i] = 0; + for(int j = 0 ; j < accumX; j++) + result[((it*volume+v)*extra+et)*site_size+i] += + h_partial_block[(v*site_size+i)*accumX+j]; + } + } else { + for(size_t v = 0 ; v < volume*t_step; v++) + for(int i = 0 ; i < site_size; i++) + result[((it*volume+v)*extra+et)*site_size+i] += + h_partial_block[v*site_size+i]; + } + } + } + + exit: + hostFree(h_partial_block, alloc_size*sizeof(FloatC)); + cudaFree(d_partial_block); + cudaFree(listGammas.array); +} + +template +void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, + int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac){ +#ifdef PLEGMA_NUCLEON_3PF_FIX_SINK + if(gammas.size() <= 0) + PLEGMA_error("Error the container of gamma matrices cannot be zero"); + if(gammas.size() > N_SPINS*N_SPINS) + PLEGMA_error("Error maximum number of gamma matrices is 16"); + + bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); + int site_size = N_DIMS*(N_DIMS-1)*gammas.size(); + + if(isZfac) + site_size *= N_SPINS*N_SPINS*N_COLS*N_COLS; + + if(corr.getSiteSize() != site_size) + PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); + + site_size = gammas.size(); + ProfileStruct ps(HGC_localVolume3D, (runFT==true) ? site_size*sizeof(Float2) : 0); + int myLocalT = corr.localT(); + int maxLocalT = myLocalT; + MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); + ps.max_volume = HGC_localVolume3D*maxLocalT; + ps.tune_globally = true; + + Float2 *result = NULL; + if(runFT) + hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); + else + result = (Float2 *) corr.H_elem(); + + tune( ps, "threep_twoD", threep_twoD_host, + ps, result, corr, vect1, vect2, signProps, gauge, gammas,false); + run( ps, "threep_twoD", threep_twoD_host, + ps, result, corr, vect1, vect2, signProps, gauge, gammas,isZfac); + + if(runFT) { + MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(corr.H_elem()), + MPI_SUM, HGC_spaceComm); + hostFree(result, corr.getTotalSize()*sizeof(Float2)); + } +#else + PLEGMA_error("You must enable PLEGMA_NUCLEON_3PF_FIX_SINK\n"); +#endif +} + +template void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac); +template void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac); + diff --git a/lib/kernels/PLEGMA_topocharge.cuh b/lib/kernels/PLEGMA_topocharge.cuh index 2ea83e4a..09e8e448 100644 --- a/lib/kernels/PLEGMA_topocharge.cuh +++ b/lib/kernels/PLEGMA_topocharge.cuh @@ -195,11 +195,11 @@ static Float calcTopoCharge(gaugeTex gaugeTex, TOPO_CHARGE_DEF charge_de calcTopChClovDef_kernel<<>>( gaugeTex, d_partial_Q ); break; } - checkCudaError(); + checkQudaError(); cudaMemcpy(h_partial_Q, d_partial_Q , gridDim.x * sizeof(Float) , cudaMemcpyDeviceToHost); cudaFree(d_partial_Q); - checkCudaError(); + checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++){ Q += h_partial_Q[i]; @@ -259,7 +259,7 @@ static Float calcPlaqClovDef(gaugeTex gaugeTex){ cudaMemcpy(h_partial_Plaq, d_partial_Plaq , gridDim.x * sizeof(Float) , cudaMemcpyDeviceToHost); cudaFree(d_partial_Plaq); - checkCudaError(); + checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++) Plaq += h_partial_Plaq[i]; diff --git a/lib/kernels/PLEGMA_u1gauge_utils.cuh b/lib/kernels/PLEGMA_u1gauge_utils.cuh index 4de156be..d857492f 100644 --- a/lib/kernels/PLEGMA_u1gauge_utils.cuh +++ b/lib/kernels/PLEGMA_u1gauge_utils.cuh @@ -29,5 +29,5 @@ void constFieldU1(u1gauge2 G, int mu, int nu, Float exparg, int xnu_0){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (G.volume() + blockDim.x -1)/blockDim.x , 1 , 1); constField_kernel<<>>(G,mu,nu,exparg,xnu_0); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_vector_utils.cuh b/lib/kernels/PLEGMA_vector_utils.cuh index 3d3983c3..81895d31 100644 --- a/lib/kernels/PLEGMA_vector_utils.cuh +++ b/lib/kernels/PLEGMA_vector_utils.cuh @@ -49,7 +49,7 @@ static void apply_gamma_vector(LEFTRIGHT LR,vector2 inOut,GAMMAS r){ apply_gamma_vector_kernel<<>>(inOut, r); break; } - checkCudaError(); + checkQudaError(); } template @@ -143,7 +143,7 @@ static void copy_to_QUDA(FloatIn* in, ColorSpinorField &qudaVec, bool isEven){ else PLEGMA_error("Precision %d not supported", qudaVec.Precision()); - checkCudaError(); + checkQudaError(); } template @@ -208,7 +208,7 @@ static void copy_from_QUDA(FloatOut* out, ColorSpinorField &qudaVec, bool isEven else PLEGMA_error("Precision %d not supported", qudaVec.Precision()); - checkCudaError(); + checkQudaError(); } template @@ -261,10 +261,10 @@ static void compute_rms(const PLEGMA_Vector3D &vec, std::vector &lis int *d_listR2 = nullptr; Float *d_absPsi = nullptr; if(listR2.size() != absPsi.size()) PLEGMA_error("List sizes should match"); - cudaMalloc((void**)&d_listR2, listR2.size() * sizeof(int)); checkCudaError(); - cudaMalloc((void**)&d_absPsi, absPsi.size() * sizeof(Float)); checkCudaError(); - cudaMemcpy(d_listR2,listR2.data(), listR2.size() * sizeof(int), cudaMemcpyHostToDevice); checkCudaError(); - cudaMemset(d_absPsi,0,absPsi.size() * sizeof(Float)); checkCudaError(); + cudaMalloc((void**)&d_listR2, listR2.size() * sizeof(int)); checkQudaError(); + cudaMalloc((void**)&d_absPsi, absPsi.size() * sizeof(Float)); checkQudaError(); + cudaMemcpy(d_listR2,listR2.data(), listR2.size() * sizeof(int), cudaMemcpyHostToDevice); checkQudaError(); + cudaMemset(d_absPsi,0,absPsi.size() * sizeof(Float)); checkQudaError(); thrust::counting_iterator first(0); thrust::counting_iterator last = first + HGC_localVolume3D; typedef thrust::device_ptr > DpF2; @@ -274,7 +274,7 @@ static void compute_rms(const PLEGMA_Vector3D &vec, std::vector &lis zipTplIntDev2 z1 = thrust::make_zip_iterator(thrust::make_tuple(first,y)); zipTplIntDev2 z2 = thrust::make_zip_iterator(thrust::make_tuple(last,y+HGC_localVolume3D)); thrust::for_each(z1,z2,computeRMS(sourceposition[0],sourceposition[1],sourceposition[2],listR2.size(),d_listR2,d_absPsi)); - cudaMemcpy(absPsi.data(), d_absPsi, absPsi.size() * sizeof(Float), cudaMemcpyDeviceToHost); checkCudaError(); + cudaMemcpy(absPsi.data(), d_absPsi, absPsi.size() * sizeof(Float), cudaMemcpyDeviceToHost); checkQudaError(); cudaFree(d_listR2); cudaFree(d_absPsi); } @@ -297,5 +297,5 @@ template static void mulGV_k(vector2 Vo, su3_2 u, vector2 Vi){ ProfileStruct ps(Vo.volume()); tuneAndRun(ps,"mulGV_kernel", mulGV_kernel, Vo, u, Vi); - checkCudaError(); + checkQudaError(); } diff --git a/plegma/qLoops.cpp b/plegma/qLoops.cpp index 0183788d..dabbd4e7 100644 --- a/plegma/qLoops.cpp +++ b/plegma/qLoops.cpp @@ -240,7 +240,7 @@ int main(int argc, char **argv) long int iorder = std::get<3>(eigSol->getEigVals()[i]); double *eigVec = eigSol->getEigVecs() + iorder*eigSol->getSize_per_Vec()*2; cudaMemcpy(phi.D_elem(), eigVec, eigSol->getBytes_per_Vec(), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); if(oneDLoops || twoDLoops) qloops_std.oneEnd_trick(phi,phi,tmp,qLtmp,gauge,-1./eigVal,true); //standard one-end trick else qloops_std.oneEnd_trick(phi,phi,-1./eigVal,true); //standard one-end trick diff --git a/plegma/qLoops_WilsonLine.cpp b/plegma/qLoops_WilsonLine.cpp index a6734b9e..46cd3135 100644 --- a/plegma/qLoops_WilsonLine.cpp +++ b/plegma/qLoops_WilsonLine.cpp @@ -182,7 +182,7 @@ int main(int argc, char **argv) long int iorder = std::get<3>(eigSol->getEigVals()[i]); double *eigVec = eigSol->getEigVecs() + iorder*eigSol->getSize_per_Vec()*2; cudaMemcpy(phi.D_elem(), eigVec, eigSol->getBytes_per_Vec(), cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); phi_r.copy(phi); qloops_std.oneEnd_trick_wilsonLine(phi,phi_r,-1./eigVal,gaugeStout,ft_std); D->apply(phi_r,phi); diff --git a/utils/PLEGMA_eigSolver.cpp b/utils/PLEGMA_eigSolver.cpp index 63e361bf..4391668f 100644 --- a/utils/PLEGMA_eigSolver.cpp +++ b/utils/PLEGMA_eigSolver.cpp @@ -151,7 +151,7 @@ static void applyOperator(double *out, double *in, int size_per_Vec){ void EigSolver::applyOperator(double *out, double *in){ #endif cudaMemcpy(d_in->D_elem(),in,bytes_per_Vec,cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); if(!G_isACC) dOp->apply(*d_out,*d_in); else{ @@ -189,7 +189,7 @@ void EigSolver::applyOperator(double *out, double *in){ } } cudaMemcpy(out,d_out->D_elem(),bytes_per_Vec,cudaMemcpyDeviceToHost); - checkCudaError(); + checkQudaError(); } #if defined(HAVE_PRIMME) @@ -428,7 +428,7 @@ void EigSolver::computeEigVals(){ for(int j = 0 ; j < p.NeV; j++){ double one[2] = {1.,0.}; cudaMemcpy(tmp1->D_elem(),ptr_tmp,bytes_per_Vec,cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); dOp->apply(*tmp2,*tmp1); std::complex eval = cuBLAS::dot(size_per_Vec, tmp1->D_elem(), tmp2->D_elem(), HGC_fullComm); cuBLAS::scal(size_per_Vec,-eval.real(),tmp1->D_elem()); @@ -494,7 +494,7 @@ void EigSolver::dumpEvalsVdagG5V(std::string filename){ double* ptr_tmp = h_eigVecs; for (int j = 0; j < p.NeV; ++j) { cudaMemcpy(g5V.D_elem(),ptr_tmp,bytes_per_Vec,cudaMemcpyHostToDevice); - checkCudaError(); + checkQudaError(); V.copy(g5V); g5V.apply_gamma(G5); std::complex res = cuBLAS::dot(size_per_Vec, V.D_elem(), g5V.D_elem(),HGC_fullComm); diff --git a/utils/QUDA_params.cpp b/utils/QUDA_params.cpp index 8eceb993..2d20c314 100644 --- a/utils/QUDA_params.cpp +++ b/utils/QUDA_params.cpp @@ -151,8 +151,8 @@ void setMultigridParam(QudaMultigridParam &mg_param) { inv_param.Ls = 1; - inv_param.sp_pad = 0; - inv_param.cl_pad = 0; +// inv_param.sp_pad = 0; +// inv_param.cl_pad = 0; inv_param.cpu_prec = cpu_prec; inv_param.cuda_prec = cuda_prec; From a630d6289c91f41fa8127cd9a10b518ace3b87d6 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 18 Oct 2022 11:44:33 +0200 Subject: [PATCH 08/25] remove artificial files --- lib/kernels/PLEGMA_threep_local_stochastic.cu | 184 ------------ lib/kernels/PLEGMA_threep_twoD_stochastic.cu | 264 ------------------ 2 files changed, 448 deletions(-) delete mode 100644 lib/kernels/PLEGMA_threep_local_stochastic.cu delete mode 100644 lib/kernels/PLEGMA_threep_twoD_stochastic.cu diff --git a/lib/kernels/PLEGMA_threep_local_stochastic.cu b/lib/kernels/PLEGMA_threep_local_stochastic.cu deleted file mode 100644 index eced1caf..00000000 --- a/lib/kernels/PLEGMA_threep_local_stochastic.cu +++ /dev/null @@ -1,184 +0,0 @@ -#include -#include -#include -#include -#include - -using namespace plegma; -template -struct KernelArr {T* array; int size;}; - -template -__global__ void threep_local_stochastic_device(Float2* block2, - vectorTex prop1Tex, vectorTex prop2Tex, - KernelArr listGammas, - int it, int time_step, int maxT, int4 source, - int signProps, bool runFT, tex_mom_list moms){ - int grid3D = gridDim.x/time_step; - int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; - int tid = blockIdx.x/grid3D; - // this takes into account the case where the source is in the local lattice - // and we need to start from it when we go over maxT - int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; - int vid = sid3D + t*DGC_localVolume3D; - - Float2 accum[N_SPINS*N_SPINS]; // max value of gammas - #pragma unroll - for(int i = 0; i < N_SPINS*N_SPINS; i++) - accum[i]=0; - - if (sid3D < DGC_localVolume3D){ - /*Float2 prop1[N_SPINS][N_SPINS][N_COLS][N_COLS]; - Float2 prop2[N_SPINS][N_SPINS][N_COLS][N_COLS]; - Float2 R[N_SPINS][N_SPINS]; - prop1Tex.get(prop1,vid); - prop2Tex.get(prop2,vid); - partial_trace_mul_Prop_Prop(R,prop1,prop2); - - for(int iop = 0; iop < listGammas.size; iop++){ - int opId=listGammas.array[iop]; - if (signProps >0){ - accum[iop]=trace_gamma_S(opId,TMP,R); - } - else if (signProps<0){ - accum[iop]=trace_gamma_S(opId,TMM,R); - } - else{ - accum[iop]=trace_gamma_S(opId,NOROT,R); - } - }*/ - } - - int source_pos[3] = {source.x, source.y, source.z}; - - if(runFT){ - extern __shared__ int ext_shared_cache[]; - Float2 *shared_cache = (Float2 *) ext_shared_cache; - fourier_transform_3D(block2, accum, shared_cache, listGammas.size, sid3D, source_pos, moms, 0, +1, time_step, tid); - } else{ - if (sid3D < DGC_localVolume3D) - for(int iop = 0; iop < listGammas.size; iop++) - block2[(tid*DGC_localVolume3D + sid3D)*listGammas.size +iop] = accum[iop]; - } -} - -template -static void threep_local_stochastic_host(ProfileStruct &ps, Float2 *result, - PLEGMA_Correlator &corr, - PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, - int signProps, std::vector& gammas ){ - - int t_size = corr.localT(); if(t_size==0) return; - int maxT = corr.endT() - corr.startT(); - int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); - - bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); - size_t volume = corr.getVolSize()/t_size; - - - size_t size = corr.getTotalSize()/t_size*time_step; - int site_size = corr.getSiteSize(); - int4 source = corr.getSource(); - auto moms = corr.getTexMomList(); - - KernelArr listGammas; - listGammas.size = gammas.size(); - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); - - if(HGC_verbosity > 2) - if(corr.hasSource()) - printf("time_step = %d, t_size = %d, maxT = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", time_step, t_size, maxT, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); - - size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; - - Float2 *h_partial_block = NULL; - Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); - hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); - - auto propTex1 = toTexture(prop1); -// auto propTex2 = toTexture(prop2); - - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; - for(int it=0; it < t_size; it+=time_step) { - int t_step = std::min(t_size-it, time_step); - dim3 grid = ps.tp.grid; - grid.x = (grid.x/time_step)*t_step; -/* threep_local_stochastic_device - <<>> - (d_partial_block, *propTex1, *propTex2, listGammas, it, t_step, maxT, source, signProps, runFT, *moms);*/ - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - - cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - - if(runFT==true){ - int accumX = ps.tp.grid.x/time_step; - for(size_t v = 0 ; v < volume*t_step; v++) - for(int i = 0 ; i < site_size; i++) { - result[(it*volume+v)*site_size+i] = 0; - for(int j = 0 ; j < accumX; j++) - result[(it*volume+v)*site_size+i] += - h_partial_block[(v*site_size+i)*accumX+j]; - } - } else { - for(size_t v = 0 ; v < volume*t_step; v++) - for(int i = 0 ; i < site_size; i++) - result[(it*volume+v)*site_size+i] += - h_partial_block[v*site_size+i]; - } - } - - exit: - hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); -} - -template -void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas) { -#ifdef PLEGMA_NUCLEON_3PF_FIX_SINK - if(gammas.size() <= 0) - PLEGMA_error("Error the container of gamma matrices cannot be zero"); - if(gammas.size() > N_SPINS*N_SPINS) - PLEGMA_error("Error maximum number of gamma matrices is 16"); - - bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); - int site_size = gammas.size(); - - if(corr.getSiteSize() != site_size) - PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); - - site_size = gammas.size(); - ProfileStruct ps(HGC_localVolume3D, (runFT==true) ? site_size*sizeof(Float2) : 0); - int myLocalT = corr.localT(); - int maxLocalT = myLocalT; - MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); - ps.max_volume = HGC_localVolume3D*maxLocalT; - ps.tune_globally = true; - - Float2 *result = NULL; - if(runFT) - hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); - else - result = (Float2 *) corr.H_elem(); - - tune( ps, "threep_local_stochastic", threep_local_stochastic_host, - ps, result, corr, prop1, prop2, signProps, gammas); - run( ps, "threep_local_stochastic", threep_local_stochastic_host, - ps, result, corr, prop1, prop2, signProps, gammas); - - if(runFT) { - MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(corr.H_elem()), - MPI_SUM, HGC_spaceComm); - hostFree(result, corr.getTotalSize()*sizeof(Float2)); - } -#else - PLEGMA_error("You must enable PLEGMA_NUCLEON_3PF_FIX_SINK\n"); -#endif -} - -template void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas); -template void threep_local_stochastic(PLEGMA_Correlator &corr, PLEGMA_Vector& prop1, PLEGMA_Vector& prop2, int signProps, std::vector& gammas); diff --git a/lib/kernels/PLEGMA_threep_twoD_stochastic.cu b/lib/kernels/PLEGMA_threep_twoD_stochastic.cu deleted file mode 100644 index 1a98a17e..00000000 --- a/lib/kernels/PLEGMA_threep_twoD_stochastic.cu +++ /dev/null @@ -1,264 +0,0 @@ -#include -#include -#include -#include -#include -#include - -using namespace plegma; -template -struct KernelArr {T* array; int size;}; - -template -__global__ void threep_twoD_device(Float2* block2, - vectorTex vect1Tex, vectorTex vect2Tex, - gaugeTex gaugeTex, KernelArr listGammas, - int it, int time_step, int maxT, int4 source, - int signProps, bool runFT, tex_mom_list moms, - int dir1, int dir2){ - int grid3D = gridDim.x/time_step; - int sid3D = (blockIdx.x % grid3D)*blockDim.x + threadIdx.x; - int tid = blockIdx.x/grid3D; - // this takes into account the case where the source is in the local lattice - // and we need to start from it when we go over maxT - int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; - int vid = sid3D + t*DGC_localVolume3D; - - Float2 accum[N_SPINS*N_SPINS]; - #pragma unroll - for(int i = 0; i < N_SPINS*N_SPINS; i++) - accum[i]=0; - - if (sid3D < DGC_localVolume3D){ - Float2 vect1[N_SPINS][N_COLS]; - Float2 vect2[N_SPINS][N_COLS]; - Float2 R[N_SPINS][N_SPINS]; - Float2 su3_1[N_COLS][N_COLS]; - Float2 su3_2[N_COLS][N_COLS]; - - // BEGIN REGION - // This has been generated by python/generate_derivative.py - // The order has been optmized such that 18 data accesses have been commented out - // + term x, x, x+dir1, x+dir1+dir2 - vect1Tex.get(vect1,vid); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1,dir2); - partial_trace_mul_Vec_G1_G2_Vec(R,vect1,vect2,su3_1,su3_2); - - // - term x, x, x+dir1-dir2^, x+dir1-dir2 - /*vect1Tex.get(vect1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); vect2Tex.get(vect2,vid,dir1,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x+dir2, x+dir2, x+dir1^, x+dir1 - vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x-dir2, x-dir2, x+dir1-dir2, x+dir1 - vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*vect2Tex.get(vect2,vid,dir1);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x, x-dir1^, x-dir1, x-dir1+dir2 - vect1Tex.get(vect1,vid); gaugeTex.get(su3_1,dir1,vid,dir1); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x, x-dir1^, x-dir1-dir2^, x-dir1-dir2 - /*vect1Tex.get(vect1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); vect2Tex.get(vect2,vid,dir1,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x+dir2, x-dir1+dir2^, x-dir1^, x-dir1 - vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); vect2Tex.get(vect2,vid,dir1); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x-dir2, x-dir1-dir2^, x-dir1-dir2, x-dir1 - vect1Tex.get(vect1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*vect2Tex.get(vect2,vid,dir1);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x+dir1, x^, x, x+dir2 - vect1Tex.get(vect1,vid,dir1); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid); vect2Tex.get(vect2,vid,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x+dir1, x^, x-dir2^, x-dir2 - /*vect1Tex.get(vect1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir2); vect2Tex.get(vect2,vid,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x-dir1, x-dir1, x-dir2^, x-dir2 - vect1Tex.get(vect1,vid,dir1); gaugeTex.get(su3_1,dir1,vid,dir1); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*vect2Tex.get(vect2,vid,dir2);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x-dir1, x-dir1, x, x+dir2 - /*vect1Tex.get(vect1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid); vect2Tex.get(vect2,vid,dir2); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x+dir1+dir2, x+dir2^, x^, x - vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ vect2Tex.get(vect2,vid); - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x-dir1+dir2, x-dir1+dir2, x^, x - vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ /*vect2Tex.get(vect2,vid);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // - term x+dir1-dir2, x-dir2^, x-dir2, x - vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir2); /*vect2Tex.get(vect2,vid);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // + term x-dir1-dir2, x-dir1-dir2, x-dir2, x - vect1Tex.get(vect1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*vect2Tex.get(vect2,vid);*/ - partial_trace_mul_Prop_G1_G2_Prop(R,vect1,vect2,su3_1,su3_2); - - // END REGION - - for(int iop = 0; iop < listGammas.size; iop++){ - int opId=listGammas.array[iop]; - if(notZfac) accum[iop] = 0.0625*((signProps > 0) ? trace_gamma_S(opId,TMP,R) : trace_gamma_S(opId,TMM,R)); - else accum[iop] = trace_gamma_S(opId,NOROT,R); - } - } - - int site_size = listGammas.size; - int source_pos[3] = {source.x, source.y, source.z}; - if(runFT){ - extern __shared__ int ext_shared_cache[]; - Float2 *shared_cache = (Float2 *) ext_shared_cache; - fourier_transform_3D(block2, accum, shared_cache, site_size, sid3D, source_pos, moms, 0, +1, time_step, tid); - } else{ - if (sid3D < DGC_localVolume3D) - for(int iop = 0; iop < site_size; iop++) - block2[(tid*DGC_localVolume3D + sid3D)*site_size+iop] = accum[iop]; - } -} - -template -static void threep_twoD_host(ProfileStruct &ps, Float2 *result, PLEGMA_Correlator &corr, - PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, - int signProps, PLEGMA_Gauge& gauge, std::vector& gammas){ - - int t_size = corr.localT(); if(t_size==0) return; - int maxT = corr.endT() - corr.startT(); - int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); - bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); - size_t volume = corr.getVolSize()/t_size; - int extra=N_DIMS*(N_DIMS-1); - if(isZfac) extra*=N_SPINS*N_SPINS*N_COLS*N_COLS; - size_t size = corr.getTotalSize()/extra/t_size*time_step; - int site_size = corr.getSiteSize()/extra; - int4 source = corr.getSource(); - auto moms = corr.getTexMomList(); - - KernelArr listGammas; - listGammas.size = gammas.size(); - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); - - if(HGC_verbosity > 2) - if(corr.hasSource()) - printf("time_step = %d, t_size = %d, maxT = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", time_step, t_size, maxT, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); - - size_t alloc_size = (runFT==true)? (size * (ps.tp.grid.x/time_step) ) : size; - - Float2 *h_partial_block = NULL; - Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); - hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); - - auto vectorTex1 = toTexture(vect1); - auto vectorTex2 = toTexture(vect2); - auto gaugetex = toTexture(gauge); - - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; - for(int it=0; it < t_size; it+=time_step) { - for(int et=0; et < extra; et++) { - int dir1 = (et/(N_DIMS-1)) % N_DIMS; - int dir2 = et % (N_DIMS-1); - if(dir2>=dir1) dir2++; - if(isZfac) { - int tt = et/(N_DIMS*(N_DIMS-1)); - mu=tt/N_SPINS/N_COLS/N_COLS; - nu=(tt/N_COLS/N_COLS)%N_SPINS; - c1=(tt/N_COLS)%N_COLS; - c2=tt%N_COLS; - } - int t_step = std::min(t_size-it, time_step); - dim3 grid = ps.tp.grid; - grid.x = (grid.x/time_step)*t_step; - threep_twoD_device - <<>> - (d_partial_block, *vectorTex1, *vectorTex2, *gaugetex, listGammas, it, t_step, maxT, - source, signProps, runFT, *moms, dir1,dir2,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - - if(runFT==true){ - int accumX = ps.tp.grid.x/time_step; - for(size_t v = 0 ; v < volume*t_step; v++) - for(int i = 0 ; i < site_size; i++) { - result[((it*volume+v)*extra+et)*site_size+i] = 0; - for(int j = 0 ; j < accumX; j++) - result[((it*volume+v)*extra+et)*site_size+i] += - h_partial_block[(v*site_size+i)*accumX+j]; - } - } else { - for(size_t v = 0 ; v < volume*t_step; v++) - for(int i = 0 ; i < site_size; i++) - result[((it*volume+v)*extra+et)*site_size+i] += - h_partial_block[v*site_size+i]; - } - } - } - - exit: - hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); -} - -template -void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, - int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac){ -#ifdef PLEGMA_NUCLEON_3PF_FIX_SINK - if(gammas.size() <= 0) - PLEGMA_error("Error the container of gamma matrices cannot be zero"); - if(gammas.size() > N_SPINS*N_SPINS) - PLEGMA_error("Error maximum number of gamma matrices is 16"); - - bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); - int site_size = N_DIMS*(N_DIMS-1)*gammas.size(); - - if(isZfac) - site_size *= N_SPINS*N_SPINS*N_COLS*N_COLS; - - if(corr.getSiteSize() != site_size) - PLEGMA_error("Correlator siteSize do not match: %d != %d\n", corr.getSiteSize(), site_size); - - site_size = gammas.size(); - ProfileStruct ps(HGC_localVolume3D, (runFT==true) ? site_size*sizeof(Float2) : 0); - int myLocalT = corr.localT(); - int maxLocalT = myLocalT; - MPI_Allreduce( &myLocalT, &maxLocalT, 1, MPI_Type(maxLocalT), MPI_MAX, HGC_fullComm); - ps.max_volume = HGC_localVolume3D*maxLocalT; - ps.tune_globally = true; - - Float2 *result = NULL; - if(runFT) - hostMalloc(result, corr.getTotalSize()*sizeof(Float2)); - else - result = (Float2 *) corr.H_elem(); - - tune( ps, "threep_twoD", threep_twoD_host, - ps, result, corr, vect1, vect2, signProps, gauge, gammas,false); - run( ps, "threep_twoD", threep_twoD_host, - ps, result, corr, vect1, vect2, signProps, gauge, gammas,isZfac); - - if(runFT) { - MPI_Allreduce(result, corr.H_elem(), corr.getTotalSize()*2, MPI_Type(corr.H_elem()), - MPI_SUM, HGC_spaceComm); - hostFree(result, corr.getTotalSize()*sizeof(Float2)); - } -#else - PLEGMA_error("You must enable PLEGMA_NUCLEON_3PF_FIX_SINK\n"); -#endif -} - -template void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac); -template void threep_twoD(PLEGMA_Correlator &corr, PLEGMA_Propagator& vect1, PLEGMA_Propagator& vect2, int signProps, PLEGMA_Gauge& gauge, std::vector& gammas, bool isZfac); - From 643b1c0e06377f80dcf64a154fb767cf0dcfaf25 Mon Sep 17 00:00:00 2001 From: Simone Bacchio Date: Wed, 19 Oct 2022 11:31:33 +0300 Subject: [PATCH 09/25] Correcting time_step --- lib/kernels/PLEGMA_QWF.cuh | 4 ++-- lib/kernels/PLEGMA_baryons.cuh | 2 +- lib/kernels/PLEGMA_mesons.cuh | 4 ++-- lib/kernels/PLEGMA_threep_staple.cu | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/kernels/PLEGMA_QWF.cuh b/lib/kernels/PLEGMA_QWF.cuh index 0a87ec36..66e58843 100644 --- a/lib/kernels/PLEGMA_QWF.cuh +++ b/lib/kernels/PLEGMA_QWF.cuh @@ -133,7 +133,7 @@ void contract_TMDWF_mesons_trick_zfac_host( ProfileStruct &ps,PLEGMA_Propagator< int extra = N_SPINS*N_SPINS*N_COLS*N_COLS; int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); size_t size = corr.getTotalSize()/extra/t_size*time_step; size_t volume = corr.getVolSize()/t_size; @@ -207,7 +207,7 @@ void contract_TMDWF_mesons_zfac_host( ProfileStruct &ps,PLEGMA_Propagator *result, int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); size_t volume = corr.getVolSize()/t_size; int extra=1; From e2d4fc3e05ffe0496c3461d34c404a578bd52451 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 12:10:44 +0200 Subject: [PATCH 10/25] defining QUDA_PRECISION --- CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b771bf05..25b10c93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,16 @@ endif() FIND_LIBRARY(QUDA_LIB quda ${QUDA_HOME}/lib) include_directories(SYSTEM ${QUDA_HOME}/include) +set(QUDA_PRECISION + "14" + CACHE STRING "which precisions to instantiate in QUDA (4-bit number - double, single, half, quarter)") +set(QUDA_RECONSTRUCT + "7" + CACHE STRING "which reconstructs to instantiate in QUDA (3-bit number - 18, 13/12, 9/8)") +add_definitions(-DQUDA_PRECISION=${QUDA_PRECISION}) +add_definitions(-DQUDA_RECONSTRUCT=${QUDA_RECONSTRUCT}) + + set(QUDA_SRC "" CACHE PATH "path to QUDA src directory") if("${QUDA_SRC}" STREQUAL "") message( FATAL_ERROR "QUDA_SRC must be defined" ) From 6f0a8c8f8bcb82a98770bce6f9682376efb82de1 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 12:11:49 +0200 Subject: [PATCH 11/25] including checkQudaError in PLEGMA_BLAS.h --- include/PLEGMA_BLAS.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/PLEGMA_BLAS.h b/include/PLEGMA_BLAS.h index b0e692db..af5cee1a 100644 --- a/include/PLEGMA_BLAS.h +++ b/include/PLEGMA_BLAS.h @@ -13,6 +13,7 @@ #include #include +#include #pragma once enum OPER_MATR_BLAS {NOTRANS, TRANS, DAGGER}; namespace cBLAS{ From b439586bee5aa69306fdb7b4e24fbcabf3db8d0c Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 12:14:51 +0200 Subject: [PATCH 12/25] including get_stream for creating process id --- lib/PLEGMA_Field.cu | 10 +++++----- lib/kernels/PLEGMA_kernel_tuner.cuh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index d487a745..271bee50 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -225,8 +225,8 @@ void PLEGMA_Field::create_host(){ template void PLEGMA_Field::create_device(){ - d_elem=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, Bytes_total_plus_ghost()); -// cudaMalloc((void**)&d_elem,Bytes_total_plus_ghost()); +// d_elem=(Float *)device_malloc(Bytes_total_plus_ghost()); + cudaMalloc((void**)&d_elem,Bytes_total_plus_ghost()); if(checkErr) checkQudaError(); #ifdef DEVICE_MEMORY_REPORT // device memory in MB @@ -671,7 +671,7 @@ template void PLEGMA_Field::shift(PLEGMA_Field &Fin, short dirOr1, short dirOr2, short dirOr3){ // we have to make sure that we have the ghost assert(dirOr1!=dirOr2 && dirOr2!=dirOr3 && dirOr1!=dirOr3); - /Fin.communicateGhost(-1, DIR_BOTH, FIRST_VERTEX); + Fin.communicateGhost(-1, DIR_BOTH, FIRST_VERTEX); shiftField(Fin,*this,dirOr1,dirOr2,dirOr3); } @@ -741,8 +741,8 @@ void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign) int D3D4 = mom.size(); int V = D3D4 == 3 ? HGC_localVolume3D : HGC_localVolume; Float2 *x; - x=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, V*2*sizeof(Float)); -// cudaMalloc((void**)&x, V*2*sizeof(Float)); + //x=((Float2) *)device_malloc(V*2*sizeof(Float)); + cudaMalloc((void**)&x, V*2*sizeof(Float)); cudaMemset((void*) x,0,V*2*sizeof(Float)); if(checkErr) checkQudaError(); std::vector momF(mom.begin(), mom.end()); diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index e99f0a6f..c236cb9a 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -274,7 +274,7 @@ void PLEGMA_kernel_tuner::run(){ launchKernel(ps.tp.grid,ps.tp.block,ps.tp.shared_bytes,0); #else if(!ps.tuned) ps.tp = tuneLaunch(*this, QUDA_TUNE_NO, (QudaVerbosity) HGC_verbosity); - launchKernel(ps.tp,0); + launchKernel(ps.tp,get_stream(0)); #endif checkQudaError(); } From b5f46e526e0e05e87ca3f77f3f7b2352e5151bdb Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 12:20:38 +0200 Subject: [PATCH 13/25] adding the location of get_stream --- lib/kernels/PLEGMA_kernel_tuner.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index c236cb9a..9b82aa6a 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -274,7 +274,7 @@ void PLEGMA_kernel_tuner::run(){ launchKernel(ps.tp.grid,ps.tp.block,ps.tp.shared_bytes,0); #else if(!ps.tuned) ps.tp = tuneLaunch(*this, QUDA_TUNE_NO, (QudaVerbosity) HGC_verbosity); - launchKernel(ps.tp,get_stream(0)); + launchKernel(ps.tp,target::cuda::get_stream(0)); #endif checkQudaError(); } From c2560b9ae4b3e7b4adb7b9fabff45bcafe33d1ab Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 12:22:19 +0200 Subject: [PATCH 14/25] updating QUDA_interface.cpp --- utils/QUDA_interface.cpp | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/utils/QUDA_interface.cpp b/utils/QUDA_interface.cpp index 55bc1dac..a35157c4 100644 --- a/utils/QUDA_interface.cpp +++ b/utils/QUDA_interface.cpp @@ -197,12 +197,14 @@ QUDA_solver::QUDA_solver(double mu) { solver = Solver::create(*solverParam, *M, *MSloppy, *MPre, *MPre, *profiler); - ColorSpinorParam cpuParam(NULL, inv_param, HGC_localL, pc_solution, + quda::lat_dim_t X = {HGC_localL[0], HGC_localL[1], HGC_localL[2], HGC_localL[3]}; + + ColorSpinorParam cpuParam(NULL, inv_param, X, pc_solution, inv_param.input_location); - ColorSpinorParam cudaParam(cpuParam, inv_param); + ColorSpinorParam cudaParam(cpuParam, inv_param,inv_param.input_location); cudaParam.create = QUDA_ZERO_FIELD_CREATE; - b = new cudaColorSpinorField(cudaParam); - x = new cudaColorSpinorField(cudaParam); + b = new ColorSpinorField(cudaParam); + x = new ColorSpinorField(cudaParam); profiler->TPSTOP(QUDA_PROFILE_TOTAL); profiler->Print(); @@ -360,7 +362,7 @@ void QUDA_solver::UpdateSolver() profiler->TPRESET(); } -cudaColorSpinorField *QUDA_solver::solve(cudaColorSpinorField * rhs){ +ColorSpinorField *QUDA_solver::solve(ColorSpinorField * rhs){ profiler->TPSTART(QUDA_PROFILE_TOTAL); ColorSpinorField *in = NULL; ColorSpinorField *out = NULL; @@ -375,7 +377,7 @@ cudaColorSpinorField *QUDA_solver::solve(cudaColorSpinorField * rhs){ template -cudaColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn){ +ColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn){ bool flag_eo=false; if( inv_param.matpc_type == QUDA_MATPC_EVEN_EVEN ) flag_eo = true; @@ -401,8 +403,8 @@ void QUDA_solver::solve(PLEGMA_Vector &vectorOut, PLEGMA_Vector &v template void QUDA_solver::solve(PLEGMA_Vector &vectorOut, PLEGMA_Vector &vectorIn); template void QUDA_solver::solve(PLEGMA_Vector &vectorOut, PLEGMA_Vector &vectorIn); -template cudaColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn); -template cudaColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn); +template ColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn); +template ColorSpinorField *QUDA_solver::solve(PLEGMA_Vector &vectorIn); template void QUDA_solver::runOneIter(PLEGMA_Vector &vectorOut, PLEGMA_Vector &vectorIn){ @@ -432,13 +434,14 @@ QUDA_dirac::QUDA_dirac(QudaDslashType dslashType): if (dParam.gauge == nullptr) PLEGMA_error("Gauge field not allocated"); if (dParam. clover == nullptr && ((inv_param.dslash_type == QUDA_CLOVER_WILSON_DSLASH) || (inv_param.dslash_type == QUDA_TWISTED_CLOVER_DSLASH))) PLEGMA_error("Clover field not allocated"); D = Dirac::create(dParam); + quda::lat_dim_t X = {HGC_localL[0], HGC_localL[1], HGC_localL[2], HGC_localL[3]}; - ColorSpinorParam cpuParam(nullptr, inv_param, HGC_localL, false, + ColorSpinorParam cpuParam(nullptr, inv_param, X, false, inv_param.input_location); - ColorSpinorParam cudaParam(cpuParam, inv_param); + ColorSpinorParam cudaParam(cpuParam, inv_param,inv_param.input_location); cudaParam.create = QUDA_ZERO_FIELD_CREATE; - in = new cudaColorSpinorField(cudaParam); - out = new cudaColorSpinorField(cudaParam); + in = new ColorSpinorField(cudaParam); + out = new ColorSpinorField(cudaParam); if(in->SiteSubset() != QUDA_FULL_SITE_SUBSET || out->SiteSubset() != QUDA_FULL_SITE_SUBSET) PLEGMA_error("cudaColorSpinorField should be a full vector for this class"); } From eb7fb72bcc5057c3f85afd9b42c1650adacef97e Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 21:00:36 +0200 Subject: [PATCH 15/25] removing non-longer existed parameters in inv_param --- utils/QUDA_params.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/QUDA_params.cpp b/utils/QUDA_params.cpp index 2d20c314..0da580f9 100644 --- a/utils/QUDA_params.cpp +++ b/utils/QUDA_params.cpp @@ -331,8 +331,8 @@ void setInvertParam(QudaInvertParam &inv_param) { inv_param.Ls = 1; - inv_param.sp_pad = 0; - inv_param.cl_pad = 0; +// inv_param.sp_pad = 0; +// inv_param.cl_pad = 0; inv_param.cpu_prec = cpu_prec; inv_param.cuda_prec = cuda_prec; From 6772da0cdf667db1f7ff596d3f0a710aa5dd6364 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 21:01:42 +0200 Subject: [PATCH 16/25] using device::get_stream instead of target::cuda::get_stream --- lib/kernels/PLEGMA_kernel_tuner.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index 9b82aa6a..a6eecfbd 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -274,7 +274,7 @@ void PLEGMA_kernel_tuner::run(){ launchKernel(ps.tp.grid,ps.tp.block,ps.tp.shared_bytes,0); #else if(!ps.tuned) ps.tp = tuneLaunch(*this, QUDA_TUNE_NO, (QudaVerbosity) HGC_verbosity); - launchKernel(ps.tp,target::cuda::get_stream(0)); + launchKernel(ps.tp,device::get_stream(0)); #endif checkQudaError(); } From 6d9382eda5f4fb1eb5a9198e1b33d98ca33f8db0 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 21:02:22 +0200 Subject: [PATCH 17/25] restoring compilation --- lib/kernels/PLEGMA_scattreductions.cuh | 40 ++++++++++++-------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/lib/kernels/PLEGMA_scattreductions.cuh b/lib/kernels/PLEGMA_scattreductions.cuh index 3ffd25b3..0e4547ec 100644 --- a/lib/kernels/PLEGMA_scattreductions.cuh +++ b/lib/kernels/PLEGMA_scattreductions.cuh @@ -50,8 +50,8 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); -// cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); - d_partial_block=device_malloc(alloc_size*sizeof(Float2)); + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); +// d_partial_block=device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. auto error= qudaGetLastError(); if(error != QUDA_SUCCESS) { @@ -66,17 +66,15 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator listGammas.size = gammas.size(); if (gammas.size()==0){ - listGammas.array=device_malloc(sizeof(GAMMAS_SCATT)); +// listGammas.array=device_malloc(sizeof(GAMMAS_SCATT)); +// checkQudaError(); + cudaMalloc((void**)&listGammas.array, sizeof(GAMMAS_SCATT)); checkQudaError(); - } - -// cudaMalloc((void**)&listGammas.array, sizeof(GAMMAS_SCATT)); -// checkCudaError(); } else{ - listGammas.array=device_malloc(gammas.size()*sizeof(GAMMAS_SCATT)); -// cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS_SCATT)); +// listGammas.array=device_malloc(gammas.size()*sizeof(GAMMAS_SCATT)); + cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS_SCATT)); auto error= qudaGetLastError(); if(error != QUDA_SUCCESS) { errorQuda("Failed to clear error state %s\n", qudaGetLastErrorString().c_str()); @@ -85,7 +83,7 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator return; } qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS_SCATT), qudaMemcpyHostToDevice); - auto error= qudaGetLastError(); + error= qudaGetLastError(); if(error != QUDA_SUCCESS) { errorQuda("Failed to copy from host to device %s\n", qudaGetLastErrorString().c_str()); PLEGMA_printf("Error in allocating %d\n",gammas.size()*sizeof(GAMMAS_SCATT)); @@ -107,15 +105,15 @@ static void V_reductions_host( ProfileStruct &ps, VRED V, PLEGMA_ScattCorrelator ps.tp.grid.x = grid.x; //Syncronize (maybe useles) and look for errors (without stopping) - cudaDeviceSynchronize(); - error=cudaPeekAtLastError(); - if(error != cudaSuccess) { PLEGMA_printf("Error after V_kernels_wrapper, it=%d tsize=%d error=%d string%s\n",it,t_size,error, cudaGetErrorString(error)); break;} + //cudaDeviceSynchronize(); + //error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { PLEGMA_printf("Error after V_kernels_wrapper, it=%d tsize=%d error=%d string%s\n",it,t_size,error, cudaGetErrorString(error)); break;} //copy partial summed 3dfourier back to host d_partial -> h_partial (device->host) cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); - if(error != cudaSuccess) { PLEGMA_printf("Error after copying back partial_block, it=%d\n",it); break;} + //error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { PLEGMA_printf("Error after copying back partial_block, it=%d\n",it); break;} //perform intermediate sum between results of different blocks with same timeslice for(size_t tslicexmom = 0 ; tslicexmom< N_moms*std::min(t_size-it, time_step); tslicexmom++){ @@ -374,9 +372,9 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - d_partial_block=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__,alloc_size*sizeof(Float2)); +// d_partial_block=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__,alloc_size*sizeof(Float2)); -// cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. cudaError_t error=cudaPeekAtLastError(); @@ -391,13 +389,13 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator listGammas_i.size = gammas_i.size(); listGammas_f.size = gammas_f.size(); - listGammas_i.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_i.size()*sizeof(GAMMAS_SCATT)); +// listGammas_i.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_i.size()*sizeof(GAMMAS_SCATT)); - listGammas_f.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_f.size()*sizeof(GAMMAS_SCATT)); +// listGammas_f.array=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gammas_f.size()*sizeof(GAMMAS_SCATT)); -// cudaMalloc((void**)&listGammas_i.array, gammas_i.size()*sizeof(GAMMAS_SCATT)); -// cudaMalloc((void**)&listGammas_f.array, gammas_f.size()*sizeof(GAMMAS_SCATT)); + cudaMalloc((void**)&listGammas_i.array, gammas_i.size()*sizeof(GAMMAS_SCATT)); + cudaMalloc((void**)&listGammas_f.array, gammas_f.size()*sizeof(GAMMAS_SCATT)); checkQudaError(); cudaMemcpy(listGammas_i.array, gammas_i.data(), gammas_i.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); cudaMemcpy(listGammas_f.array, gammas_f.data(), gammas_f.size()*sizeof(GAMMAS_SCATT), cudaMemcpyHostToDevice); From e93e1e8cb27f34af75c2625083fa7a59247fbd9a Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 21:02:52 +0200 Subject: [PATCH 18/25] restoring compilation --- lib/kernels/PLEGMA_su3field.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kernels/PLEGMA_su3field.cuh b/lib/kernels/PLEGMA_su3field.cuh index e0ae4963..43678ee6 100644 --- a/lib/kernels/PLEGMA_su3field.cuh +++ b/lib/kernels/PLEGMA_su3field.cuh @@ -174,8 +174,8 @@ static void sum_real_trace_host(ProfileStruct& ps, PLEGMA_Su3field &su3M int gridDimX = ps.tp.grid.x; hostMalloc(h_partial_sum, gridDimX * sizeof(Float) ); -// cudaMalloc((void**)&d_partial_sum, gridDimX * sizeof(Float)); - d_partial_sum=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gridDimX * sizeof(Float)); + cudaMalloc((void**)&d_partial_sum, gridDimX * sizeof(Float)); +// d_partial_sum=quda::device_malloc_(__func__, quda::file_name(__FILE__), __LINE__, gridDimX * sizeof(Float)); sum_real_trace_kernel<<>>(toField2(su3M), d_partial_sum); From 4e186f53876d2c77426f5ef20f8ad1177cd2f539 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Tue, 25 Oct 2022 21:27:53 +0200 Subject: [PATCH 19/25] renaming clover to clover_term --- lib/kernels/PLEGMA_topocharge.cuh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/kernels/PLEGMA_topocharge.cuh b/lib/kernels/PLEGMA_topocharge.cuh index 09e8e448..fe064ddb 100644 --- a/lib/kernels/PLEGMA_topocharge.cuh +++ b/lib/kernels/PLEGMA_topocharge.cuh @@ -10,7 +10,7 @@ using namespace plegma; //device function that computes the clover term! save a clover extracted from gaugetex (dir1,dir2,sid) into a C matrix //COMMENTS: check with clover-definded plaquette OK! template -__device__ void clover( Float2 C[N_COLS][N_COLS], gaugeTex &gaugeTex, int dir1, int dir2, int sid ) { +__device__ void clover_term( Float2 C[N_COLS][N_COLS], gaugeTex &gaugeTex, int dir1, int dir2, int sid ) { Float2 G1[N_COLS][N_COLS], G2[N_COLS][N_COLS], G3[N_COLS][N_COLS], G4[N_COLS][N_COLS], P[N_COLS][N_COLS]; @@ -91,8 +91,8 @@ static __global__ void calcTopChClovDef_kernel(gaugeTex gaugeTex, Float FloatG tr_aux = 0. ; #pragma unroll for(int i=0; i<3; i++) { - clover( clov1, gaugeTex, dir0[i], dir1[i], sid ); - clover( clov2, gaugeTex, dir2[i], dir3[i], sid ); + clover_term( clov1, gaugeTex, dir0[i], dir1[i], sid ); + clover_term( clov2, gaugeTex, dir2[i], dir3[i], sid ); tr_aux = trace_mul_ImG_ImG( clov1, clov2 ); @@ -226,7 +226,7 @@ static __global__ void calcPlaqClovDef_kernel(gaugeTex gaugeTex, Float * for(int dir1=0; dir1( clov_tmp ); } } From 92bc63b8bd14e5c2c8d4383383f402152a827442 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Wed, 26 Oct 2022 15:49:14 +0200 Subject: [PATCH 20/25] correcting typo in documentation --- include/PLEGMA_ScattCorrelator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/PLEGMA_ScattCorrelator.h b/include/PLEGMA_ScattCorrelator.h index 71184ead..be3d3558 100644 --- a/include/PLEGMA_ScattCorrelator.h +++ b/include/PLEGMA_ScattCorrelator.h @@ -190,7 +190,7 @@ namespace plegma { void T1( std::vector &Gammas_i, std::vector &Gammas_f, PLEGMA_Propagator &S1, PLEGMA_Propagator &S2, PLEGMA_Propagator &S3 ); /** * @brief performs T2 type reduction to compute baryon 2pt functions - * T2_{alpha,beta}=\epsilon_{a,b,c}\epsilon_{l,m,n}S2^{c,l}_{alpha,beta}\Gamma_{i}_{alpha0,alpha1}S2^{b,m}_{beta0,alpha1}\Gamma_{f}_{beta0,beta1}S3^{a,n}_{beta1,alpha0} + * T2_{alpha,beta}=\epsilon_{a,b,c}\epsilon_{l,m,n}S1^{c,l}_{alpha,beta}\Gamma_{i}_{alpha0,alpha1}S2^{b,m}_{beta0,alpha1}\Gamma_{f}_{beta0,beta1}S3^{a,n}_{beta1,alpha0} * @param std::vector &Gammas_i: list of gammas at the source for the contractions * @param std::vector &Gammas_f: list of gammas at the sink for the contractions * @param PLEGMA_Propagator &S1: propagator to the sink spin from the source spin index From cbf44129343af2e69e599d7c434c5dd8c67873d9 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Thu, 27 Oct 2022 11:01:35 +0200 Subject: [PATCH 21/25] getting compiled --- lib/PLEGMA.cu | 6 ++++-- lib/kernels/PLEGMA_kernel_tuner.cuh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/PLEGMA.cu b/lib/PLEGMA.cu index 2b3aa552..e5b7f77a 100644 --- a/lib/PLEGMA.cu +++ b/lib/PLEGMA.cu @@ -7,14 +7,16 @@ #include #include #include -#include #include +#include //#define TIMING_REPORT using namespace plegma; +using namespace quda; //extern Topology *default_topo; std::vector HDF5::open_files; Communicator &get_current_communicator(); +Communicator default_t; void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ HGC_hold_exit = false; @@ -28,7 +30,7 @@ void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ for(int i = 0 ; i < N_DIMS ; i++) HGC_localL[i] = localL[i]; - HGC_default_topo = get_current_communicator().default_topo; + HGC_default_topo = default_t.comm_default_topology();//get_current_communicator().default_topo;//default_topo;// get_current_communicator().default_topo; HGC_verbosity = verbosity; for(int i = 0 ; i < N_DIMS ; i++) { HGC_nProc[i] = nProcs[i]; diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index a6eecfbd..67b3c810 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -264,8 +264,8 @@ void PLEGMA_kernel_tuner::apply(const qudaStream_t &stream){ #endif } -//template -//void PLEGMA_kernel_tuner::apply(){ apply(0); } +template +void PLEGMA_kernel_tuner::apply(){ apply(device::get_stream(0)); } template void PLEGMA_kernel_tuner::run(){ From 61463d85eedb18427ce05b5141ca949b8b6e56c9 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Wed, 2 Nov 2022 11:32:22 +0100 Subject: [PATCH 22/25] replacing functions needed default topology --- lib/PLEGMA.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/PLEGMA.cu b/lib/PLEGMA.cu index e5b7f77a..7616882f 100644 --- a/lib/PLEGMA.cu +++ b/lib/PLEGMA.cu @@ -16,7 +16,9 @@ using namespace quda; //extern Topology *default_topo; std::vector HDF5::open_files; Communicator &get_current_communicator(); -Communicator default_t; +//Communicator default_t; +//extern Topology *default_topo; + void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ HGC_hold_exit = false; @@ -30,7 +32,7 @@ void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ for(int i = 0 ; i < N_DIMS ; i++) HGC_localL[i] = localL[i]; - HGC_default_topo = default_t.comm_default_topology();//get_current_communicator().default_topo;//default_topo;// get_current_communicator().default_topo; +// HGC_default_topo = get_current_communicator().comm_default_topology();//default_tcomm_default_topology();//get_current_communicator().default_topo;//default_topo;// get_current_communicator().default_topo; HGC_verbosity = verbosity; for(int i = 0 ; i < N_DIMS ; i++) { HGC_nProc[i] = nProcs[i]; @@ -161,7 +163,7 @@ void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ #endif for(int i= 0 ; i < N_DIMS ; i++) - HGC_procPosition[i] = comm_coords(HGC_default_topo)[i]; + HGC_procPosition[i] = comm_coord(i); // copying globals to device HGC_global_vars.copyToDevice(); From 64e1600c79c13853d0952e62d952dd8c377253a6 Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Wed, 2 Nov 2022 11:33:46 +0100 Subject: [PATCH 23/25] replacing functions needed default topology 2 --- lib/PLEGMA_Field.cu | 170 +++++++++++++++++----------------- lib/PLEGMA_ScattCorrelator.cu | 5 +- 2 files changed, 89 insertions(+), 86 deletions(-) diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index 271bee50..e0325506 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -11,6 +11,8 @@ #include #include #include +#include +#include using namespace plegma; #define DEVICE_MEMORY_REPORT @@ -1007,94 +1009,94 @@ void PLEGMA_Field::absorbTimeslice(PLEGMA_Field &srcfield, int glo //check dimensions - int my_it = global_it - comm_coords(HGC_default_topo)[3] * HGC_localL[3]; - bool is_myIt = (my_it >= 0) && ( my_it < HGC_localL[3] ); - int V3 = HGC_localVolume/HGC_localL[3]; - int V4 = HGC_localVolume; - Float *pointer_src = NULL; - Float *pointer_dst = NULL; - - - for(int i = 0 ; i < this->field_length; i++){ - if( forcetozero ) - cudaMemset( this->d_elem + i*V4*2, 0, V4*2*sizeof(Float)); - if(is_myIt){ - pointer_dst = (this->d_elem + i*V4*2 + my_it*V3*2); - pointer_src = (srcfield.D_elem() + i*V4*2 + my_it*V3*2); - cudaMemcpy(pointer_dst, pointer_src, V3*2 * sizeof(Float), cudaMemcpyDeviceToDevice); - } - } - comm_barrier(); - checkQudaError(); -} + int my_it = global_it - comm_coord(3) * HGC_localL[3]; + bool is_myIt = (my_it >= 0) && ( my_it < HGC_localL[3] ); + int V3 = HGC_localVolume/HGC_localL[3]; + int V4 = HGC_localVolume; + Float *pointer_src = NULL; + Float *pointer_dst = NULL; + + + for(int i = 0 ; i < this->field_length; i++){ + if( forcetozero ) + cudaMemset( this->d_elem + i*V4*2, 0, V4*2*sizeof(Float)); + if(is_myIt){ + pointer_dst = (this->d_elem + i*V4*2 + my_it*V3*2); + pointer_src = (srcfield.D_elem() + i*V4*2 + my_it*V3*2); + cudaMemcpy(pointer_dst, pointer_src, V3*2 * sizeof(Float), cudaMemcpyDeviceToDevice); + } + } + comm_barrier(); + checkQudaError(); + } -template -void PLEGMA_Field::TrFmunuSu3FmunuSu3(PLEGMA_Fmunu &Fl, std::pair munu_l, PLEGMA_Su3field &Wl, - PLEGMA_Fmunu &Fr, std::pair munu_r, - PLEGMA_Su3field &Wr){ - traceMulFmunuSu3FmunuSu3_k(*this,Fl,munu_l,Wl,Fr,munu_r,Wr); -} + template + void PLEGMA_Field::TrFmunuSu3FmunuSu3(PLEGMA_Fmunu &Fl, std::pair munu_l, PLEGMA_Su3field &Wl, + PLEGMA_Fmunu &Fr, std::pair munu_r, + PLEGMA_Su3field &Wr){ + traceMulFmunuSu3FmunuSu3_k(*this,Fl,munu_l,Wl,Fr,munu_r,Wr); + } -template -void PLEGMA_Field::trPmunu(PLEGMA_Gauge &gauge, std::pair munu){ - gauge.communicateSideGhost(); - trPmunu_k(*this,gauge,munu); -} + template + void PLEGMA_Field::trPmunu(PLEGMA_Gauge &gauge, std::pair munu){ + gauge.communicateSideGhost(); + trPmunu_k(*this,gauge,munu); + } -template class PLEGMA_Field; -template class PLEGMA_Field; -// Forcing initialization of the following cases -template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); -template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); -template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); -template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); -template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); - -// field3D <- field4D -template -void PLEGMA_Field3D::absorb(const PLEGMA_Field &field, int global_it, bool broadcast){ - if(global_it >= HGC_totalL[3]) PLEGMA_error("The global time slice you provided exceed the temporal extent\n"); - assert(field.Field_length() == this->Field_length()); - int my_it = global_it - HGC_procPosition[3] * HGC_localL[3]; - this->activeTimeSlice = (my_it >= 0) && ( my_it < HGC_localL[3] ); - size_t V3 = HGC_localVolume3D*2; - size_t V4 = HGC_localVolume*2; - Float *pointer_src = NULL; - Float *pointer_dst = NULL; - for(int i = 0; i < this->Field_length(); i++) { - pointer_dst = (this->D_elem() + i*V3); - if(this->activeTimeSlice) { - pointer_src = (field.D_elem() + i*V4 + my_it*V3); - cudaMemcpy(pointer_dst, pointer_src, V3 * sizeof(Float), cudaMemcpyDeviceToDevice); - } - if (broadcast == true){ - int time_rank=global_it/HGC_localL[3]; - Float *temp=(Float *)malloc(sizeof(Float)*V3); - cudaMemcpy(temp, pointer_dst, V3* sizeof(Float), cudaMemcpyDeviceToHost); - MPI_Bcast(temp, V3 , MPI_Type(), time_rank, HGC_timeComm); - cudaMemcpy(pointer_dst, temp, V3* sizeof(Float), cudaMemcpyHostToDevice); - free(temp); - } - if (broadcast == false && !(this->activeTimeSlice)){ - cudaMemset(pointer_dst, 0, V3 * sizeof(Float)); - } - } - checkQudaError(); -} + template class PLEGMA_Field; + template class PLEGMA_Field; + // Forcing initialization of the following cases + template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); + template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); + template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); + template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + template void PLEGMA_Field::mulMomentumPhases(std::vector mom, int sign); + + // field3D <- field4D + template + void PLEGMA_Field3D::absorb(const PLEGMA_Field &field, int global_it, bool broadcast){ + if(global_it >= HGC_totalL[3]) PLEGMA_error("The global time slice you provided exceed the temporal extent\n"); + assert(field.Field_length() == this->Field_length()); + int my_it = global_it - HGC_procPosition[3] * HGC_localL[3]; + this->activeTimeSlice = (my_it >= 0) && ( my_it < HGC_localL[3] ); + size_t V3 = HGC_localVolume3D*2; + size_t V4 = HGC_localVolume*2; + Float *pointer_src = NULL; + Float *pointer_dst = NULL; + for(int i = 0; i < this->Field_length(); i++) { + pointer_dst = (this->D_elem() + i*V3); + if(this->activeTimeSlice) { + pointer_src = (field.D_elem() + i*V4 + my_it*V3); + cudaMemcpy(pointer_dst, pointer_src, V3 * sizeof(Float), cudaMemcpyDeviceToDevice); + } + if (broadcast == true){ + int time_rank=global_it/HGC_localL[3]; + Float *temp=(Float *)malloc(sizeof(Float)*V3); + cudaMemcpy(temp, pointer_dst, V3* sizeof(Float), cudaMemcpyDeviceToHost); + MPI_Bcast(temp, V3 , MPI_Type(), time_rank, HGC_timeComm); + cudaMemcpy(pointer_dst, temp, V3* sizeof(Float), cudaMemcpyHostToDevice); + free(temp); + } + if (broadcast == false && !(this->activeTimeSlice)){ + cudaMemset(pointer_dst, 0, V3 * sizeof(Float)); + } + } + checkQudaError(); + } -template -std::complex PLEGMA_Field3D::dot(PLEGMA_Field3D &fieldIn){ - // TODO: need to think about appropriate communicator - if (HGC_localVolume != HGC_totalVolume) - PLEGMA_warning("3D Vector dot might not work with multiple MPI ranks\n"); - return cuBLAS::dot(this->total_length*this->field_length, this->d_elem, fieldIn.D_elem(), HGC_fullComm); -} + template + std::complex PLEGMA_Field3D::dot(PLEGMA_Field3D &fieldIn){ + // TODO: need to think about appropriate communicator + if (HGC_localVolume != HGC_totalVolume) + PLEGMA_warning("3D Vector dot might not work with multiple MPI ranks\n"); + return cuBLAS::dot(this->total_length*this->field_length, this->d_elem, fieldIn.D_elem(), HGC_fullComm); + } -template class PLEGMA_Field3D; + template class PLEGMA_Field3D; template class PLEGMA_Field3D; diff --git a/lib/PLEGMA_ScattCorrelator.cu b/lib/PLEGMA_ScattCorrelator.cu index c6d410a0..d1e98d2c 100644 --- a/lib/PLEGMA_ScattCorrelator.cu +++ b/lib/PLEGMA_ScattCorrelator.cu @@ -6,6 +6,7 @@ #include #include #include +#include using namespace plegma; bool gammas_isSym( std::vector &Gammas ){ @@ -376,7 +377,7 @@ Float *PLEGMA_ScattCorrelator::get_source_time_slice(){ memcpy(ptr, this->H_elem()+t_source_local*size_timeslice, sizeof(Float)*size_timeslice); int coords[4]; for(int i = 0 ; i < N_DIMS; i++) coords[i] = this->getSource()[i] / HGC_localL[i]; - int rankHas = comm_rank_from_coords(HGC_default_topo, coords); + int rankHas = quda::comm_rank_from_coords(HGC_default_topo, coords); int mpiErr = MPI_Bcast(ptr, size_timeslice, MPI_Type(), rankHas, HGC_fullComm); if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Bcast failed with error %d\n", mpiErr); @@ -3458,7 +3459,7 @@ void PLEGMA_ScattCorrelator::absorbTimeslice(PLEGMA_ScattCorrelator= HGC_totalL[3]) PLEGMA_error("The global time slice you provided exceed the temporal extent\n"); - int my_it = global_it - comm_coords(HGC_default_topo)[3] * HGC_localL[3]; + int my_it = global_it - comm_coord(3) * HGC_localL[3]; bool is_myIt = (my_it >= 0) && ( my_it < HGC_localL[3] ); From bf16b3ddb13b14bd781ddca6c1be37449813e35a Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Sun, 6 Nov 2022 21:12:34 +0100 Subject: [PATCH 24/25] replacing checkCudaError with checkQudaError --- lib/PLEGMA_Field.cu | 44 ++++--------------------- lib/PLEGMA_Propagator.cu | 2 +- lib/kernels/PLEGMA_field_utils.cuh | 2 +- lib/kernels/PLEGMA_propagator_utils.cuh | 4 +-- 4 files changed, 10 insertions(+), 42 deletions(-) diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index efbc2885..5a70b1b1 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -1093,46 +1093,14 @@ void PLEGMA_Field3D::absorb(const PLEGMA_Field &field, int global_ checkQudaError(); } -// field3D <- field4D + template -void PLEGMA_Field3D::absorb(const PLEGMA_Field &field, int global_it, bool broadcast){ - if(global_it >= HGC_totalL[3]) PLEGMA_error("The global time slice you provided exceed the temporal extent\n"); - assert(field.Field_length() == this->Field_length()); - int my_it = global_it - HGC_procPosition[3] * HGC_localL[3]; - this->activeTimeSlice = (my_it >= 0) && ( my_it < HGC_localL[3] ); - size_t V3 = HGC_localVolume3D*2; - size_t V4 = HGC_localVolume*2; - Float *pointer_src = NULL; - Float *pointer_dst = NULL; - for(int i = 0; i < this->Field_length(); i++) { - pointer_dst = (this->D_elem() + i*V3); - if(this->activeTimeSlice) { - pointer_src = (field.D_elem() + i*V4 + my_it*V3); - cudaMemcpy(pointer_dst, pointer_src, V3 * sizeof(Float), cudaMemcpyDeviceToDevice); - } - if (broadcast == true){ - int time_rank=global_it/HGC_localL[3]; - Float *temp=(Float *)malloc(sizeof(Float)*V3); - cudaMemcpy(temp, pointer_dst, V3* sizeof(Float), cudaMemcpyDeviceToHost); - MPI_Bcast(temp, V3 , MPI_Type(), time_rank, HGC_timeComm); - cudaMemcpy(pointer_dst, temp, V3* sizeof(Float), cudaMemcpyHostToDevice); - free(temp); - } - if (broadcast == false && !(this->activeTimeSlice)){ - cudaMemset(pointer_dst, 0, V3 * sizeof(Float)); - } - } - checkCudaError(); +std::complex PLEGMA_Field3D::dot(PLEGMA_Field3D &fieldIn){ + // TODO: need to think about appropriate communicator + if (HGC_localVolume != HGC_totalVolume) + PLEGMA_warning("3D Vector dot might not work with multiple MPI ranks\n"); + return cuBLAS::dot(this->total_length*this->field_length, this->d_elem, fieldIn.D_elem(), HGC_fullComm); } ->>>>>>> origin/softfunctionTMDPDFs - - template - std::complex PLEGMA_Field3D::dot(PLEGMA_Field3D &fieldIn){ - // TODO: need to think about appropriate communicator - if (HGC_localVolume != HGC_totalVolume) - PLEGMA_warning("3D Vector dot might not work with multiple MPI ranks\n"); - return cuBLAS::dot(this->total_length*this->field_length, this->d_elem, fieldIn.D_elem(), HGC_fullComm); - } template class PLEGMA_Field3D; template class PLEGMA_Field3D; diff --git a/lib/PLEGMA_Propagator.cu b/lib/PLEGMA_Propagator.cu index 5701f2fd..50b57804 100644 --- a/lib/PLEGMA_Propagator.cu +++ b/lib/PLEGMA_Propagator.cu @@ -199,7 +199,7 @@ void PLEGMA_Propagator::PropmulVVdag(PLEGMA_Vector &vec1,PLEGMA_Ve auto vectex1 = toTexture(vec1); auto vectex2 = toTexture(vec2); prop_mul_V_Vdag(toField2(*this), *vectex1, *vectex2); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_field_utils.cuh b/lib/kernels/PLEGMA_field_utils.cuh index 9ff0ab3a..6f1b1c50 100644 --- a/lib/kernels/PLEGMA_field_utils.cuh +++ b/lib/kernels/PLEGMA_field_utils.cuh @@ -371,5 +371,5 @@ static void SU3Trace_k(PLEGMA_Field &f,PLEGMA_Su3field &su3field assert(f.checkVolume(su3field)); ProfileStruct ps(su3field.Total_length()); tuneAndRun(ps,"SU3Trace_kernel",SU3Trace_kernel,f.D_elem(),toField2(su3field)); - checkCudaError(); + checkQudaError(); } diff --git a/lib/kernels/PLEGMA_propagator_utils.cuh b/lib/kernels/PLEGMA_propagator_utils.cuh index 79d2178b..e46aeece 100644 --- a/lib/kernels/PLEGMA_propagator_utils.cuh +++ b/lib/kernels/PLEGMA_propagator_utils.cuh @@ -168,6 +168,6 @@ template static void prop_mul_V_Vdag(prop2 prop, vectorTex& vectex1, vectorTex& vectex2){ ProfileStruct ps(prop.volume()); run(ps, "prop_mul_V_Vdag_kernel", prop_mul_V_Vdag_kernel, prop, vectex1, vectex2); - checkCudaError(); + checkQudaError(); } - \ No newline at end of file + From 1bfcd10ba58d73d2d9a8e8fbb28c3d20b50b749c Mon Sep 17 00:00:00 2001 From: Ferenc Pittler Date: Mon, 7 Nov 2022 10:25:20 +0100 Subject: [PATCH 25/25] correcting time_step for tetraquarks kernel --- lib/kernels/PLEGMA_bcud_tetraquarks.cu | 2 +- lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu | 2 +- lib/kernels/PLEGMA_heavy_light_tetraquarks.cu | 2 +- lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu | 2 +- lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/kernels/PLEGMA_bcud_tetraquarks.cu b/lib/kernels/PLEGMA_bcud_tetraquarks.cu index c18c8653..4c5493d6 100644 --- a/lib/kernels/PLEGMA_bcud_tetraquarks.cu +++ b/lib/kernels/PLEGMA_bcud_tetraquarks.cu @@ -68,7 +68,7 @@ void contract_tetraquarks_bcud_host(ProfileStruct &ps, int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); int4 source = corr.getSource(); size_t volume3D = corr.getVolSize()/t_size; diff --git a/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu b/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu index 3e16d6e0..1f170e77 100644 --- a/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu +++ b/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu @@ -98,7 +98,7 @@ void contract_tetraquarks_bcud_stochastic_host(ProfileStruct &ps, int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x,ps.tp.block.x); bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); int4 source = corr.getSource(); size_t volume3D = corr.getVolSize()/t_size; diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu index dc67153e..8bf5ee7c 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu @@ -68,7 +68,7 @@ void contract_tetraquarks_host(ProfileStruct &ps, int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); int4 source = corr.getSource(); size_t volume3D = corr.getVolSize()/t_size; diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu index 2df7c539..9a25b2a1 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu @@ -94,7 +94,7 @@ static void tetraquark_open_index_host(ProfileStruct &ps, Float2 *result int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace() == MOMENTUM_SPACE); size_t volume = corr.getVolSize()/t_size; size_t size = corr.getTotalSize()/t_size*time_step; diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu index 48221ad5..eab9c4af 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu @@ -98,7 +98,7 @@ void contract_tetraquarks_stochastic_host(ProfileStruct &ps, int t_size = corr.localT(); if(t_size==0) return; int maxT = corr.endT() - corr.startT(); - int time_step = ps.tp.grid.x*ps.tp.block.x/HGC_localVolume3D; + int time_step = get_time_step(ps.tp.grid.x, ps.tp.block.x); bool runFT = (corr.getCorrSpace()==MOMENTUM_SPACE); int4 source = corr.getSource(); size_t volume3D = corr.getVolSize()/t_size;