diff --git a/CMakeLists.txt b/CMakeLists.txt index 4348da2c..de804c00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,8 @@ # basic setup for cmake +if(POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) +endif() + cmake_minimum_required(VERSION 3.12 FATAL_ERROR) set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON) @@ -9,45 +13,44 @@ set(CMAKE_CXX_STANDARD 17) include(CheckIncludeFileCXX) include(CheckSymbolExists) -if(${CMAKE_VERSION} VERSION_GREATER "3.12.0") - cmake_policy(SET CMP0074 NEW) -endif() -# disable in source builds -# this is only a temporary fix, but for now we need it as cmake will -# otherwise overwrite the existing makefiles +# Disable in source builds this is only a temporary fix, but for now we need it as +# cmake will otherwise overwrite the existing makefiles. set(CMAKE_DISABLE_SOURCE_CHANGES ON) set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) # add a directory for cmake modules list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") -enable_language(C) -enable_language(CXX) -project("PLEGMA" VERSION 0.0.1 LANGUAGES) -# TODO: add authomatic download and compilation of QUDA -set(QUDA_HOME "" CACHE PATH "path to QUDA") -if("${QUDA_HOME}" STREQUAL "") - message( FATAL_ERROR "QUDA_HOME must be defined" ) + +# By default we will build DEVEL. The different build types will pass different +# flags to the compiler which may be strict or permissive on warnings, or +# very verbose at run time and/or compile time. +if(DEFINED ENV{PLEGMA_BUILD_TYPE}) + set(DEFBUILD $ENV{PLEGMA_BUILD_TYPE}) +else() + set(DEFBUILD "DEVEL") endif() -FIND_LIBRARY(QUDA_LIB quda ${QUDA_HOME}/lib) -include_directories(SYSTEM ${QUDA_HOME}/include) -set(QUDA_PRECISION - "14" - CACHE STRING "which precisions to instantiate in QUDA (4-bit number - double, single, half, quarter)") -set(QUDA_RECONSTRUCT - "7" - CACHE STRING "which reconstructs to instantiate in QUDA (3-bit number - 18, 13/12, 9/8)") -add_definitions(-DQUDA_PRECISION=${QUDA_PRECISION}) -add_definitions(-DQUDA_RECONSTRUCT=${QUDA_RECONSTRUCT}) +set(VALID_BUILD_TYPES + DEVEL + RELEASE + STRICT + DEBUG + HOSTDEBUG + SANITIZE) +set(CMAKE_BUILD_TYPE + "${DEFBUILD}" + CACHE STRING "Choose the type of build, options are: ${VALID_BUILD_TYPES}") +set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS ${VALID_BUILD_TYPES}) -set(QUDA_SRC "" CACHE PATH "path to QUDA src directory") -if("${QUDA_SRC}" STREQUAL "") - message( FATAL_ERROR "QUDA_SRC must be defined" ) -endif() +string(TOUPPER ${CMAKE_BUILD_TYPE} CHECK_BUILD_TYPE) +list(FIND VALID_BUILD_TYPES ${CHECK_BUILD_TYPE} BUILD_TYPE_VALID) +if(BUILD_TYPE_VALID LESS 0) + message(SEND_ERROR "Please specify a valid CMAKE_BUILD_TYPE type! Valid build types are:" "${VALID_BUILD_TYPES}") +endif() find_package(Git) if(GIT_FOUND) execute_process(COMMAND ${GIT_EXECUTABLE} show @@ -57,27 +60,27 @@ if(GIT_FOUND) if(${IS_GIT_REPOSIITORY} EQUAL 0) execute_process(COMMAND ${GIT_EXECUTABLE} merge-base --is-ancestor b08233a7d1cee0145e56751ae04c5542ce98b3b5 HEAD WORKING_DIRECTORY ${QUDA_SRC} - RESULT_VARIABLE QUDA_INCLUDES_IT + RESULT_VARIABLE QUDA_INCLUDES_IT OUTPUT_VARIABLE GITTAG - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE) if(${QUDA_INCLUDES_IT} EQUAL 0) add_compile_definitions(QUDA_INCLUDES_COMMIT_b08233a) endif() execute_process(COMMAND ${GIT_EXECUTABLE} merge-base --is-ancestor 775a033a84b4f085e8cf68812b89f989e6a5a816 HEAD WORKING_DIRECTORY ${QUDA_SRC} - RESULT_VARIABLE QUDA_INCLUDES_IT + RESULT_VARIABLE QUDA_INCLUDES_IT OUTPUT_VARIABLE GITTAG - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE) if(${QUDA_INCLUDES_IT} EQUAL 0) add_compile_definitions(QUDA_INCLUDES_COMMIT_775a033) endif() execute_process(COMMAND ${GIT_EXECUTABLE} merge-base --is-ancestor 55782743d1ee8d99af21e1038be11a7023650d7d HEAD - WORKING_DIRECTORY ${QUDA_SRC} - RESULT_VARIABLE QUDA_INCLUDES_IT + WORKING_DIRECTORY ${QUDA_SRC} + RESULT_VARIABLE QUDA_INCLUDES_IT OUTPUT_VARIABLE GITTAG - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE) if(${QUDA_INCLUDES_IT} EQUAL 0) add_compile_definitions(QUDA_INCLUDES_COMMIT_55782743) endif() @@ -85,15 +88,86 @@ if(GIT_FOUND) execute_process(COMMAND ${GIT_EXECUTABLE} merge-base --is-ancestor 1dec1db7a57e4fe00b14e04ba2088ca5039f8760 HEAD WORKING_DIRECTORY ${QUDA_SRC} - RESULT_VARIABLE QUDA_INCLUDES_IT + RESULT_VARIABLE QUDA_INCLUDES_IT OUTPUT_VARIABLE GITTAG - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE) if(${QUDA_INCLUDES_IT} EQUAL 0) add_compile_definitions(QUDA_INCLUDES_COMMIT_1dec1db) endif() endif() endif(GIT_FOUND) +# QUDA may be built to run using CUDA, HIP or SYCL, which we call the +# Target type. By default, the target is CUDA. +if(DEFINED ENV{PLEGMA_TARGET}) + set(DEFTARGET $ENV{PLEGMA_TARGET}) +else() + set(DEFTARGET "CUDA") +endif() + +set(VALID_TARGET_TYPES CUDA HIP SYCL) +set(PLEGMA_TARGET_TYPE + "${DEFTARGET}" + CACHE STRING "Choose the type of target, options are: ${VALID_TARGET_TYPES}") +set_property(CACHE PLEGMA_TARGET_TYPE PROPERTY STRINGS CUDA HIP) + +string(TOUPPER ${PLEGMA_TARGET_TYPE} CHECK_TARGET_TYPE) +list(FIND VALID_TARGET_TYPES ${CHECK_TARGET_TYPE} TARGET_TYPE_VALID) + +if(TARGET_TYPE_VALID LESS 0) + message(SEND_ERROR "Please specify a valid QUDA_TARGET_TYPE type! Valid target types are:" "${VALID_TARGET_TYPES}") +endif() + + +# disable in source builds +# this is only a temporary fix, but for now we need it as cmake will +# otherwise overwrite the existing makefiles +set(CMAKE_DISABLE_SOURCE_CHANGES ON) +set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) + +# add a directory for cmake modules +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +enable_language(C) +enable_language(CXX) + +# Define the project +project( + "PLEGMA" + VERSION 1.1.0 + LANGUAGES) + +# Print the configuration details to stdout +message(STATUS "") +message(STATUS "${PROJECT_NAME} ${PROJECT_VERSION} (${GITVERSION}) **") +message(STATUS "cmake version: ${CMAKE_VERSION}") +message(STATUS "Source location: ${CMAKE_SOURCE_DIR}") +message(STATUS "Build location: ${CMAKE_BINARY_DIR}") +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +message(STATUS "PLEGMA target: ${PLEGMA_TARGET_TYPE}") + +# TODO: add authomatic download and compilation of QUDA +set(QUDA_HOME "" CACHE PATH "path to QUDA") +if("${QUDA_HOME}" STREQUAL "") + message( FATAL_ERROR "QUDA_HOME must be defined" ) +endif() +FIND_LIBRARY(QUDA_LIB quda ${QUDA_HOME}/lib) +include_directories(SYSTEM ${QUDA_HOME}/include) + +set(QUDA_PRECISION + "14" + CACHE STRING "which precisions to instantiate in QUDA (4-bit number - double, single, half, quarter)") +set(QUDA_RECONSTRUCT + "7" + CACHE STRING "which reconstructs to instantiate in QUDA (3-bit number - 18, 13/12, 9/8)") +add_definitions(-DQUDA_PRECISION=${QUDA_PRECISION}) +add_definitions(-DQUDA_RECONSTRUCT=${QUDA_RECONSTRUCT}) + + +set(QUDA_SRC "" CACHE PATH "path to QUDA src directory") +if("${QUDA_SRC}" STREQUAL "") + message( FATAL_ERROR "QUDA_SRC must be defined" ) +endif() + include(ExternalProject) find_package(Boost) @@ -116,15 +190,9 @@ else() include_directories(${DOWNLOAD_DIR}/BoostPP/include) endif() -set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS DEVEL RELEASE STRICT DEBUG HOSTDEBUG DEVICEDEBUG) - # using dirs in LD_LIBRARY_PATh for searching for libraries string(REPLACE ":" ";" LIBRARY_DIRS $ENV{LD_LIBRARY_PATH}) -set(DEFAULT_GPU_ARCH sm_60) -set(GPU_ARCH ${DEFAULT_GPU_ARCH} CACHE STRING "set the GPU architecture (sm_20, sm_21, sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_70)") -set_property(CACHE GPU_ARCH PROPERTY STRINGS sm_20 sm_21 sm_30 sm_35 sm_37 sm_50 sm_52 sm_60 sm_70) - # max threads set(PLEGMA_MAX_THREADS 256 CACHE INT "maximum number of threads per block in tuner") mark_as_advanced(PLEGMA_MAX_THREADS) @@ -139,14 +207,7 @@ else() endif() -# Use CUDA textures -set(PLEGMA_TEXTURE TRUE CACHE BOOL "Wheater to use or not CUDA textures") -mark_as_advanced(PLEGMA_TEXTURE) -if(PLEGMA_TEXTURE) - add_definitions(-DPLEGMA_TEXTURE) -else() - -endif() +add_definitions(-DMULTI_GPU) # compile contractions for nucleon fix sink 3pf set(PLEGMA_NUCLEON_3PF_FIX_SINK FALSE CACHE BOOL "compile contractions for nucleon 3pf fix sink method") @@ -269,25 +330,31 @@ mark_as_advanced(PLEGMA_RUN_TESTS) ####################################################################### # we need to check for some packages find_package(PythonInterp) - -if(${CMAKE_VERSION} VERSION_GREATER 3.7.99) - find_package(CUDAWrapper) - set(USING_CUDA_LANG_SUPPORT True) - set(CMAKE_CUDA_STANDARD 17) - set(CMAKE_CUDA_STANDARD_REQUIRED True) -else() - set(CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE FILEPATH "Host side compiler used by NVCC") - mark_as_advanced(CUDA_HOST_COMPILER) - find_package(CUDA REQUIRED) - set(USING_CUDA_LANG_SUPPORT False) +if (${PLEGMA_TARGET_TYPE} STREQUAL "CUDA") + if(${CMAKE_VERSION} VERSION_GREATER 3.7.99) + find_package(CUDAWrapper) + set(USING_CUDA_LANG_SUPPORT True) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED True) + else() + set(CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE FILEPATH "Host side compiler used by NVCC") + mark_as_advanced(CUDA_HOST_COMPILER) + find_package(CUDA REQUIRED) + set(USING_CUDA_LANG_SUPPORT False) + endif() endif() -# solve compiler issues with CUDA and tuples -if( (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.5) AND (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER 9.0) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)) - message(FATAL_ERROR "This library will have compilation problems with CUDA 9.1 and gcc 6") -endif() -if( (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER 9.1) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.3) ) - message(FATAL_ERROR "This library will have compilation problems with CUDA 9.2 and gcc < 6") +if (${PLEGMA_TARGET_TYPE} STREQUAL "HIP") + if(${CMAKE_VERSION} VERSION_GREATER 3.7.99) + set(USING_HIP_LANG_SUPPORT True) + set(CMAKE_HIP_STANDARD 17) + set(CMAKE_HIP_STANDARD_REQUIRED True) + else() + set(HIP_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE FILEPATH "Host side compiler used by NVCC") + mark_as_advanced(HIP_HOST_COMPILER) + find_package(HIP REQUIRED) + set(USING_HIP_LANG_SUPPORT False) + endif() endif() # We need threads @@ -351,9 +418,9 @@ if(HDF5_FOUND) FIND_LIBRARY(PLEGMA_Z_LIB z ${LIBRARY_DIRS}) #link_libraries(${HDF5_C_LIBRARY_hdf5}) message(STATUS "HDF5 version ${HDF5_VERSION} found, include dir: ${HDF5_INCLUDE_DIRS}, libraries ${HDF5_C_LIBRARY_hdf5}") - if(NOT HDF5_IS_PARALLEL) - message(FATAL_ERROR "HDF5 must be parallel") - endif() + #if(NOT HDF5_IS_PARALLEL) + # message(FATAL_ERROR "HDF5 must be parallel") + #endif() else() message(FATAL_ERROR "HDF5 could not be located, did you try specifying -DPLEGMA_HDF5HOME?") endif() @@ -366,15 +433,6 @@ add_definitions(-DHAVE_LIME) FIND_LIBRARY(PLEGMA_LIME_LIB lime ${PLEGMA_LIMEHOME}/lib) include_directories(SYSTEM ${PLEGMA_LIMEHOME}/include) -LIST(APPEND CUDA_LIBS ${CUDA_cufft_LIBRARY} ${CUDA_curand_LIBRARY}) -LIST(APPEND CUDA_LIBS ${CUDA_cublas_LIBRARY}) -LIST(APPEND CUDA_LIBS ${CUDA_nvrtc_LIBRARY}) -LIST(APPEND CUDA_LIBS ${CUDA_nvToolsExt_LIBRARY}) - -find_package(LibDL) -LIST(APPEND CUDA_LIBS ${LIBDL_LIBRARIES}) - -add_definitions(-DMULTI_GPU) set(MPI_HOME_MANUAL "" CACHE PATH "Path to MPI for manual setting") mark_as_advanced(MPI_HOME_MANUAL) @@ -409,26 +467,10 @@ add_definitions(-DMPI_COMMS) # COMPILER OPTIONS and BUILD types include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(SYSTEM ${CUDA_INCLUDE_DIRS}) include_directories(include) include_directories(utils) include_directories(lib) -include_directories(lib/kernels) - -# GPU ARCH -STRING(REGEX REPLACE sm_ "" COMP_CAP ${GPU_ARCH}) -SET(COMP_CAP "${COMP_CAP}0") -add_definitions(-D__COMPUTE_CAPABILITY__=${COMP_CAP}) - - -# NVCC FLAGS independet off build type -if(NOT USING_CUDA_LANG_SUPPORT) - set(CUDA_NVCC_FLAGS "-std c++17 -arch=${GPU_ARCH} -ftz=true -prec-div=false -prec-sqrt=false") -else() - set(CUDA_NVCC_FLAGS "-ftz=true -prec-div=false -prec-sqrt=false") - set(CMAKE_CUDA_FLAGS "-arch=${GPU_ARCH}" CACHE STRING "Flags used by the CUDA compiler" FORCE) -endif() # some clang warnings shouds be warning even when turning warnings into errors if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -439,45 +481,6 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() endif() -## define CUDA flags when CMake < 3.8 -set(CUDA_NVCC_FLAGS_DEVEL ${CUDA_NVCC_FLAGS} -Wno-deprecated-gpu-targets -Xcompiler -Wno-unknown-pragmas,-Wno-unused-function,-Wno-unused-local-typedef,-Wno-unused-private-field -O3 -lineinfo CACHE STRING - "Flags used by the CUDA compiler during regular development builds." - FORCE ) -set(CUDA_NVCC_FLAGS_STRICT ${CUDA_NVCC_FLAGS_DEVEL} CACHE STRING - "Flags used by the CUDA compiler during strict jenkins builds." - FORCE ) -set(CUDA_NVCC_FLAGS_RELEASE ${CUDA_NVCC_FLAGS} -O3 -w CACHE STRING - "Flags used by the C++ compiler during release builds." - FORCE ) -set(CUDA_NVCC_FLAGS_HOSTDEBUG ${CUDA_NVCC_FLAGS} -g -lineinfo -DHOST_DEBUG CACHE STRING - "Flags used by the C++ compiler during host-debug builds." - FORCE ) -set(CUDA_NVCC_FLAGS_DEVICEDEBUG ${CUDA_NVCC_FLAGS} -G CACHE STRING - "Flags used by the C++ compiler during device-debug builds." - FORCE ) -set(CUDA_NVCC_FLAGS_DEBUG ${CUDA_NVCC_FLAGS} -g -DHOST_DEBUG -G CACHE STRING - "Flags used by the C++ compiler during full (host+device) debug builds." - FORCE ) - -## define CUDA flags when CMake >= 3.8 -set(CMAKE_CUDA_FLAGS_DEVEL "${CUDA_NVCC_FLAGS} -Wno-deprecated-gpu-targets -Xcompiler -Wno-unknown-pragmas,-Wno-unused-function,-Wno-unused-local-typedef,-Wno-unused-private-field -O3 -lineinfo" CACHE STRING - "Flags used by the CUDA compiler during regular development builds." - FORCE ) -set(CMAKE_CUDA_FLAGS_STRICT "${CMAKE_CUDA_FLAGS_DEVEL}" CACHE STRING - "Flags used by the CUDA compiler during strict jenkins builds." - FORCE ) -set(CMAKE_CUDA_FLAGS_RELEASE "${CUDA_NVCC_FLAGS} -O3 -w" CACHE STRING - "Flags used by the CUDA compiler during release builds." - FORCE ) -set(CMAKE_CUDA_FLAGS_HOSTDEBUG "${CUDA_NVCC_FLAGS} -g -lineinfo -DHOST_DEBUG" CACHE STRING - "Flags used by the C++ compiler during host-debug builds." - FORCE ) -set(CMAKE_CUDA_FLAGS_DEVICEDEBUG "${CUDA_NVCC_FLAGS} -G" CACHE STRING - "Flags used by the C++ compiler during device-debug builds." - FORCE ) -set(CMAKE_CUDA_FLAGS_DEBUG "${CUDA_NVCC_FLAGS} -g -DHOST_DEBUG -G" CACHE STRING - "Flags used by the C++ compiler during full (host+device) debug builds." - FORCE ) #define CXX FLAGS set(CMAKE_CXX_FLAGS_DEVEL "${OpenMP_CXX_FLAGS} -std=c++17 -O3 -Wall ${CLANG_FORCE_COLOR}" CACHE STRING @@ -542,15 +545,6 @@ mark_as_advanced(CMAKE_CXX_FLAGS_STRICT) mark_as_advanced(CMAKE_CXX_FLAGS_HOSTDEBUG) mark_as_advanced(CMAKE_CXX_FLAGS_DEVICEDEBUG) -mark_as_advanced(CUDA_NVCC_FLAGS_DEVEL) -mark_as_advanced(CUDA_NVCC_FLAGS_STRICT) -mark_as_advanced(CUDA_NVCC_FLAGS_HOSTDEBUG) -mark_as_advanced(CUDA_NVCC_FLAGS_DEVICEDEBUG) -mark_as_advanced(CMAKE_CUDA_FLAGS_DEVEL) -mark_as_advanced(CMAKE_CUDA_FLAGS_STRICT) -mark_as_advanced(CMAKE_CUDA_FLAGS_HOSTDEBUG) -mark_as_advanced(CMAKE_CUDA_FLAGS_DEVICEDEBUG) - mark_as_advanced(CMAKE_C_FLAGS_DEVEL) mark_as_advanced(CMAKE_C_FLAGS_STRICT) mark_as_advanced(CMAKE_C_FLAGS_HOSTDEBUG) diff --git a/PLEGMAConfig.cmake.in b/PLEGMAConfig.cmake.in new file mode 100644 index 00000000..01800f05 --- /dev/null +++ b/PLEGMAConfig.cmake.in @@ -0,0 +1,14 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +set(PLEGMA_TARGET_CUDA @PLEGMA_TARGET_CUDA@) +set(PLEGMA_TARGET_HIP @PLEGMA_TARGET_HIP@) + +if( PLEGMA_TARGET_CUDA ) + include(${CMAKE_CURRENT_LIST_DIR}/find_target_cuda_dependencies.cmake) +elseif(PLEGMA_TARGET_HIP ) + include(${CMAKE_CURRENT_LIST_DIR}/find_target_hip_dependencies.cmake ) +endif() + +include(${CMAKE_CURRENT_LIST_DIR}/PLEGMATargets.cmake) diff --git a/cmake/find_target_cuda_dependencies.cmake b/cmake/find_target_cuda_dependencies.cmake new file mode 100644 index 00000000..3082b9b9 --- /dev/null +++ b/cmake/find_target_cuda_dependencies.cmake @@ -0,0 +1,6 @@ +# CUDA Specific CMake + +enable_language(CUDA) + +find_dependency(CUDAToolkit REQUIRED) + diff --git a/cmake/find_target_hip_dependencies.cmake b/cmake/find_target_hip_dependencies.cmake new file mode 100644 index 00000000..c4ced99d --- /dev/null +++ b/cmake/find_target_hip_dependencies.cmake @@ -0,0 +1,20 @@ +# HIP Specific CMake +enable_language(HIP) + +if (NOT DEFINED ROCM_PATH ) + if (NOT DEFINED ENV{ROCM_PATH} ) + set(ROCM_PATH "/opt/rocm" CACHE PATH "ROCm path") + else() + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "ROCm path") + endif() +endif() + +set(CMAKE_MODULE_PATH "${ROCM_PATH}/lib/cmake" ${CMAKE_MODULE_PATH}) +find_dependency(HIP REQUIRED) +find_dependency(hipfft REQUIRED) +find_dependency(hiprand REQUIRED) +find_dependency(rocrand REQUIRED) +find_dependency(hipblas REQUIRED) +find_dependency(rocblas REQUIRED) +find_dependency(hipcub REQUIRED) +find_dependency(rocprim REQUIRED) diff --git a/include/PLEGMA_BLAS.h b/include/PLEGMA_BLAS.h index 862f758d..1bede261 100644 --- a/include/PLEGMA_BLAS.h +++ b/include/PLEGMA_BLAS.h @@ -5,13 +5,19 @@ #if defined(HAVE_MKL) #include #elif defined(HAVE_OPENBLAS) -#include +#include //#include // do not know when is needed or not #else #error Neither mkl nor openBLAS have been defined #endif +#if defined (__HIP__) +#include +#else #include +//#else +//#error Neither HIP nor NVCC have been defined +#endif #include #include #include @@ -97,160 +103,12 @@ namespace cBLAS{ } //=================================================================// +#if defined (__HIP__) +#include "target/hip/PLEGMA_hipBLAS.h" +#else +#include "target/cuda/PLEGMA_cuBLAS.h" +#endif -namespace cuBLAS{ - - template - inline void axpy(int NN, Float val[2], Float *x, Float *y){} - template<> - inline void axpy(int NN, float val[2], float *x, float *y){ - cuComplex cu_val = make_cuComplex(val[0],val[1]); - cublasStatus_t error = cublasCaxpy(HGC_cublas_handle,NN, &cu_val, (cuComplex*) x, 1, (cuComplex*) y, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCaxpy failed with error %d", error); - } - template<> - inline void axpy(int NN, double val[2], double *x, double *y){ - cuDoubleComplex cu_val = make_cuDoubleComplex(val[0],val[1]); - cublasStatus_t error = cublasZaxpy(HGC_cublas_handle, NN, &cu_val,(cuDoubleComplex*) x, 1, (cuDoubleComplex*) y, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZaxpy failed with error %d", error); - } - - //------------------------------------------------------------------ - template - inline void scal(int NN, const Float val, Float *x); - template<> - inline void scal(int NN, const float val, float *x){ - cublasStatus_t error = cublasCsscal(HGC_cublas_handle, NN, &val, (cuComplex*) x, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCsscal failed with error %d", error); - } - template<> - inline void scal(int NN, const double val, double *x){ - cublasStatus_t error = cublasZdscal(HGC_cublas_handle, NN, &val, (cuDoubleComplex*) x, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZdscal failed with error %d", error); - } - //------------------------------------------------------------------ - template - inline void cscal(int NN, const Float val[2], Float *x); - template<> - inline void cscal(int NN, const float val[2], float *x){ - cuComplex cu_val = make_cuComplex(val[0],val[1]); - cublasStatus_t error = cublasCscal(HGC_cublas_handle, NN, &cu_val, (cuComplex*) x, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCscal failed with error %d", error); - } - template<> - inline void cscal(int NN, const double val[2], double *x){ - cuDoubleComplex cu_val = make_cuDoubleComplex(val[0],val[1]); - cublasStatus_t error = cublasZscal(HGC_cublas_handle, NN, &cu_val, (cuDoubleComplex*) x, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZscal failed with error %d", error); - } - //----------------------------------------------------------------- - template - inline std::complex dot(int NN, const Float *x, const Float *y); - template<> - inline std::complex dot(int NN, const float *x, const float *y){ - cuComplex cu_res; - cublasStatus_t error = cublasCdotc(HGC_cublas_handle, NN,(cuComplex*)x, 1, (cuComplex*)y,1,&cu_res); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); - return std::complex(cu_res.x,cu_res.y); - } - template<> - inline std::complex dot(int NN, const double *x, const double *y){ - cuDoubleComplex cu_res; - cublasStatus_t error = cublasZdotc(HGC_cublas_handle, NN,(cuDoubleComplex*)x, 1, (cuDoubleComplex*)y,1,&cu_res); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZdotc failed with error %d", error); - return std::complex(cu_res.x,cu_res.y); - } - template - inline std::complex dot(int NN, const Float *x, const Float *y, MPI_Comm comm) { - if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); - std::complex result, res = cuBLAS::dot(NN, x, y); - int mpiErr = MPI_Allreduce((Float*) &res, (Float*) &result, 2, - MPI_Type(), MPI_SUM, comm); - if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); - return result; - } - //----------------------------------------------------------------- - template - inline Float norm(int NN, const Float *x); - template<> - inline float norm(int NN, const float *x){ - float res; - cublasStatus_t error = cublasScnrm2(HGC_cublas_handle, NN, (cuComplex*)x, 1, &res); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); - return res; - } - template<> - inline double norm(int NN, const double *x){ - double res; - cublasStatus_t error = cublasDznrm2(HGC_cublas_handle, NN, (cuDoubleComplex*)x, 1, &res); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); - return res; - } - template - inline Float norm(int NN, const Float *x, MPI_Comm comm) { - if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); - Float result, loc_res = std::pow(cuBLAS::norm(NN, x),2); - int mpiErr = MPI_Allreduce(&loc_res, &result, 1, MPI_Type(), MPI_SUM, - comm); - if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); - return sqrt(result); - } - - - //--------------------------------------------------------- - template - inline void gemv_(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){} - - template<> - inline void gemv_(OPER_MATR_BLAS trans, int m, int n, float alpha[2], float* A, float* x, float beta[2], float* y){ - cublasOperation_t Oper; - cuComplex cu_alpha = make_cuComplex(alpha[0],alpha[1]); - cuComplex cu_beta = make_cuComplex(beta[0],beta[1]); - switch(trans){case(NOTRANS): Oper=CUBLAS_OP_N; break; case(TRANS): Oper=CUBLAS_OP_T; break; case(DAGGER): Oper=CUBLAS_OP_C; break;} - cublasStatus_t error = cublasCgemv(HGC_cublas_handle, Oper, m, n, &cu_alpha, (cuComplex*) A, m, (cuComplex*) x, 1, &cu_beta, (cuComplex*) y, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCgemv failed with error %d", error); - } - - template<> - inline void gemv_(OPER_MATR_BLAS trans, int m, int n, double alpha[2], double* A, double* x, double beta[2], double* y){ - cublasOperation_t Oper; - cuDoubleComplex cu_alpha = make_cuDoubleComplex(alpha[0],alpha[1]); - cuDoubleComplex cu_beta = make_cuDoubleComplex(beta[0],beta[1]); - switch(trans){case(NOTRANS): Oper=CUBLAS_OP_N; break; case(TRANS): Oper=CUBLAS_OP_T; break; case(DAGGER): Oper=CUBLAS_OP_C; break;} - cublasStatus_t error = cublasZgemv(HGC_cublas_handle, Oper, m, n, &cu_alpha, (cuDoubleComplex*) A, m,(cuDoubleComplex*) x, 1, &cu_beta,(cuDoubleComplex*) y, 1); - if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZgemv failed with error %d", error); - } - - // In case of m is partitioned and trans=NOTRANS OR m is not partitioned one can use whatever trans - // Cannot work if n is partitioned - template - inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){ - gemv_(trans, m, n, alpha, A, x, beta, y); - } - // In case that m is partitioned and we take trans of dagger then reduction is needed - // Cannot work if n is partitioned - template - inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, Float *yHost, MPI_Comm comm){ - if(trans == NOTRANS) PLEGMA_error("Use gemv without MPI comm"); - if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); - cuBLAS::gemv_(trans, m, n, alpha, A, x, beta, y); - qudaMemcpy(yHost,y,n*2*sizeof(Float),qudaMemcpyDeviceToHost); - checkQudaError(); - int mpiErr = MPI_Allreduce(MPI_IN_PLACE,yHost,n*2,MPI_Type(yHost),MPI_SUM,comm); - if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); - } - - // In case that m is partitioned and we take trans of dagger then reduction is needed - // Cannot work if n is partitioned - template - inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, MPI_Comm comm){ - Float yHost[n*2]; - cuBLAS::gemv(trans,m, n, alpha, A, x, beta, y, yHost,comm); - qudaMemcpy(y,yHost,sizeof(yHost),qudaMemcpyHostToDevice); - checkQudaError(); - } -} -//=================================================================// namespace plegma{ template diff --git a/include/PLEGMA_FT.h b/include/PLEGMA_FT.h index 50777488..b7b7bd0d 100644 --- a/include/PLEGMA_FT.h +++ b/include/PLEGMA_FT.h @@ -63,11 +63,17 @@ namespace plegma { @params bool accum = false: In case we want to accumulation results from each transformation on the class buffer @params bool dimT = HGC_localL[DIM_T]: Size of the time dimension in case we want to transform only part of the vector **/ - template - PLEGMA_FT(std::vector mom, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + PLEGMA_FT(std::vector mom, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); - template - PLEGMA_FT( std::vector> &moms, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + PLEGMA_FT( std::vector> &moms, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + + PLEGMA_FT(std::vector mom, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + + PLEGMA_FT( std::vector> &moms, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + + PLEGMA_FT(std::vector mom, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); + + PLEGMA_FT( std::vector> &moms, int D3D4 = 3, bool accum = false, int dimT = HGC_localL[DIM_T]); ~PLEGMA_FT() {}; diff --git a/include/PLEGMA_Field.h b/include/PLEGMA_Field.h index 0b041191..4e41ac66 100644 --- a/include/PLEGMA_Field.h +++ b/include/PLEGMA_Field.h @@ -113,11 +113,20 @@ namespace plegma { /** @brief Creates a texture object which binds on field elements on GPU */ +#ifdef __NVCC__ cudaTextureObject_t createTexObject() const; +#elif defined (__HIP__) + hipTextureObject_t createTexObject() const; +#endif /** @brief Destroys the texture object which binds on field elements on GPU */ +#ifdef __NVCC__ void destroyTexObject(cudaTextureObject_t tex) const; +#elif defined (__HIP__) + void destroyTexObject(hipTextureObject_t tex) const; +#endif + /** * @return a pointer to access Host elements of the field */ diff --git a/include/PLEGMA_Hprobing.h b/include/PLEGMA_Hprobing.h index 71c69c7e..bb7c5f44 100644 --- a/include/PLEGMA_Hprobing.h +++ b/include/PLEGMA_Hprobing.h @@ -1,4 +1,5 @@ #pragma once +#include //#include #include @@ -101,15 +102,15 @@ namespace plegma { createElemColBlock(); createColLattice(); // if(check)checkColoring(); - cudaMalloc((void**)&d_arrVc, HGC_localVolume*sizeof(int)); + d_arrVc=(int*)device_malloc(HGC_localVolume*sizeof(int)); // checkQudaError(); - cudaMemcpy(d_arrVc, h_arrVc, HGC_localVolume*sizeof(int), cudaMemcpyHostToDevice); + qudaMemcpy(d_arrVc, h_arrVc, HGC_localVolume*sizeof(int), qudaMemcpyHostToDevice); // checkQudaError(); } ~PLEGMA_Hprobing(){ delete[] h_arrVc; delete[] arrlc; - cudaFree(d_arrVc); + device_free(d_arrVc); //checkQudaError(); } int* H_arrVc() const{return h_arrVc;} diff --git a/include/PLEGMA_Random.h b/include/PLEGMA_Random.h index 23e1917c..8f132622 100644 --- a/include/PLEGMA_Random.h +++ b/include/PLEGMA_Random.h @@ -1,5 +1,7 @@ #include -#include +#include + +//#include #ifndef _PLEGMA_RANDOM_H #define _PLEGMA_RANDOM_H @@ -14,6 +16,7 @@ namespace plegma { * XORWOW- XOR bit dependent RNG * MRG32K3a- MRG32 dependent RNG * */ +/* #if defined(XORWOW) typedef struct curandStateXORWOW cuRNGState; #elif defined(MRG32k3a) @@ -21,6 +24,7 @@ namespace plegma { #else typedef struct curandStateMRG32k3a cuRNGState; #endif +*/ enum DIST { Uniform, Normal }; /** @@ -41,13 +45,13 @@ namespace plegma { /*! @brief Backup CURAND array states initialization */ void backup(); /*! array with current curand rng state */ - __host__ __device__ __inline__ cuRNGState* State(){ return state;}; + __host__ __device__ __inline__ RNGState* State(){ return state;}; //cuRNGState *state; private: /*! array with current curand rng state */ - cuRNGState *state; + RNGState *state; /*! array for backup of current curand rng state */ - cuRNGState *backup_state; + RNGState *backup_state; /*! initial rng seed */ int seed; /*! @brief number of curand states */ @@ -73,29 +77,29 @@ namespace plegma { * */ template - inline __device__ Float PLEGMA_Random(cuRNGState &state, Float a, Float b){ + inline __device__ Float PLEGMA_Random(RNGState &state, Float a, Float b){ Float res; return res; } template<> - inline __device__ float PLEGMA_Random(cuRNGState &state, float a, float b){ - return a + (b - a) * curand_uniform(&state); + inline __device__ float PLEGMA_Random(RNGState &state, float a, float b){ + return a + (b - a) * uniform::rand(state); } template<> - inline __device__ double PLEGMA_Random(cuRNGState &state, double a, double b){ - return a + (b - a) * curand_uniform_double(&state); + inline __device__ double PLEGMA_Random(RNGState &state, double a, double b){ + return a + (b - a) * uniform::rand(state); } template<> - inline __device__ float PLEGMA_Random(cuRNGState &state, float a, float b){ - return a + b * curand_normal(&state); + inline __device__ float PLEGMA_Random(RNGState &state, float a, float b){ + return a + b * normal::rand(state); } template<> - inline __device__ double PLEGMA_Random(cuRNGState &state, double a, double b){ - return a + b * curand_normal_double(&state); + inline __device__ double PLEGMA_Random(RNGState &state, double a, double b){ + return a + b * normal::rand(state); } /** @@ -104,29 +108,29 @@ namespace plegma { * @return random number in range 0,1 * */ template - inline __device__ Float PLEGMA_Random(cuRNGState &state){ + inline __device__ Float PLEGMA_Random(RNGState &state){ Float res; return res; } template<> - inline __device__ float PLEGMA_Random(cuRNGState &state){ - return curand_uniform(&state); + inline __device__ float PLEGMA_Random(RNGState &state){ + return uniform::rand(state); } template<> - inline __device__ double PLEGMA_Random(cuRNGState &state){ - return curand_uniform_double(&state); + inline __device__ double PLEGMA_Random(RNGState &state){ + return uniform::rand(state); } template<> - inline __device__ float PLEGMA_Random(cuRNGState &state){ - return curand_normal(&state); + inline __device__ float PLEGMA_Random(RNGState &state){ + return normal::rand(state); } template<> - inline __device__ double PLEGMA_Random(cuRNGState &state){ - return curand_normal_double(&state); + inline __device__ double PLEGMA_Random(RNGState &state){ + return normal::rand(state); } } diff --git a/include/PLEGMA_Thrust.h b/include/PLEGMA_Thrust.h index 89bebb31..81ad71ea 100644 --- a/include/PLEGMA_Thrust.h +++ b/include/PLEGMA_Thrust.h @@ -1,3 +1,4 @@ +#pragma once #undef SIZE #include #include diff --git a/include/PLEGMA_global.h b/include/PLEGMA_global.h index 7ee0d3e9..49d9e486 100644 --- a/include/PLEGMA_global.h +++ b/include/PLEGMA_global.h @@ -1,8 +1,7 @@ #pragma once - //======== External libraries =========// #include -#include +////#include #include #include #include @@ -15,7 +14,10 @@ #include #include #include -#include +#ifndef __HIP__ +#include +#endif +//#include #ifdef __GNUG__ // gnu C++ compiler #include #include @@ -45,6 +47,7 @@ using namespace std::chrono_literals; #include #include #include +#include using namespace quda; namespace plegma { diff --git a/include/global/PLEGMA_global_constants.h b/include/global/PLEGMA_global_constants.h index b57b658b..53ec8b69 100644 --- a/include/global/PLEGMA_global_constants.h +++ b/include/global/PLEGMA_global_constants.h @@ -15,7 +15,11 @@ * - on host: "dtype" HGC_"name" "[s1][s2][..]" * - on device: __constant__ "dtype" DGC_"name" "[s1][s2][..]" */ - +#ifdef __HIP__ +#include +#else +#include +#endif #ifdef ADD_TO_GLOBAL #define global_host(dtype, name, ...) \ @@ -23,21 +27,33 @@ PRODUCT(__VA_ARGS__), \ &HGC_##name PARENTHESES(0,__VA_ARGS__)) +#if defined (__HIP__) #define global_both(dtype, name, ...) \ HGC_global_vars.add(#name, \ PRODUCT(__VA_ARGS__), \ &HGC_##name PARENTHESES(0,__VA_ARGS__), \ (void**) &DGC_##name) +#else +#define global_both(dtype, name, ...) \ + HGC_global_vars.add(#name, \ + PRODUCT(__VA_ARGS__), \ + &HGC_##name PARENTHESES(0,__VA_ARGS__), \ + (void**) &DGC_##name) +#endif #else #ifdef ALLOCATE #define global_host(dtype, name, ...) \ dtype HGC_##name PARENTHESES(1,__VA_ARGS__); -#ifdef __NVCC__ +#if defined ( __NVCC__ ) #define global_both(dtype, name, ...) \ dtype HGC_##name PARENTHESES(1,__VA_ARGS__); \ - __constant__ dtype DGC_##name PARENTHESES(1,__VA_ARGS__); + __constant__ dtype DGC_##name PARENTHESES(1,__VA_ARGS__); +#elif defined (__HIP__) +#define global_both(dtype, name, ...) \ + dtype HGC_##name PARENTHESES(1,__VA_ARGS__); \ + __device__ __constant__ dtype DGC_##name PARENTHESES(1,__VA_ARGS__); #else #define global_both(dtype, name, ...) \ dtype HGC_##name PARENTHESES(1,__VA_ARGS__); @@ -47,10 +63,14 @@ #define global_host(dtype, name, ...) \ extern dtype HGC_##name PARENTHESES(1,__VA_ARGS__); -#ifdef __NVCC__ +#if defined ( __NVCC__ ) #define global_both(dtype, name, ...) \ extern dtype HGC_##name PARENTHESES(1,__VA_ARGS__); \ extern __constant__ dtype DGC_##name PARENTHESES(1,__VA_ARGS__); +#elif defined ( __HIP__ ) +#define global_both(dtype, name, ...) \ + extern dtype HGC_##name PARENTHESES(1,__VA_ARGS__); \ + extern __device__ __constant__ dtype DGC_##name PARENTHESES(1,__VA_ARGS__); #else #define global_both(dtype, name, ...) \ extern dtype HGC_##name PARENTHESES(1,__VA_ARGS__); @@ -70,9 +90,6 @@ global_host(Options *, options); global_host(bool, hold_exit); // variables visible on both host and device -global_both(size_t, localVolume); -global_both(size_t, localVolume3D); -global_both(size_t, totalVolume); global_both(int, localL, N_DIMS); global_both(int, totalL, N_DIMS); global_both(int, procPosition, N_DIMS); @@ -88,6 +105,9 @@ global_both(size_t, vertexGhostVolume3D); global_both(size_t, surface3D, N_DIMS); global_both(size_t, surface2D, (N_DIMS*(N_DIMS-1))/2); global_both(size_t, surface1D, (N_DIMS*(N_DIMS-1)*(N_DIMS-2))/6); +global_both(size_t, localVolume); +global_both(size_t, localVolume3D); +global_both(size_t, totalVolume); // for mpi use global variables (host only) global_both(bool, dimBreak, N_DIMS); @@ -107,7 +127,11 @@ global_host(int, timeRank); global_host(int, timeSize); // for cublas use +#if defined (__HIP__) +global_host(hipblasHandle_t, hipblas_handle); +#else global_host(cublasHandle_t, cublas_handle); +#endif #undef global_both #undef global_host diff --git a/include/global/PLEGMA_structs.h b/include/global/PLEGMA_structs.h index 6542c86c..6a8bc463 100644 --- a/include/global/PLEGMA_structs.h +++ b/include/global/PLEGMA_structs.h @@ -1,4 +1,12 @@ //======== Some custom data struct =========// +#define HIP_CHECK(error) \ + { \ + hipError_t localError = error; \ + if ((localError != hipSuccess) && (localError != hipErrorPeerAccessAlreadyEnabled)) { \ + PLEGMA_printf("Error: %s, Code %d\n", hipGetErrorString(localError), localError); \ + PLEGMA_printf("FILE: %s, LINE %d\n", __FILE__, __LINE__ ); \ + } \ + } template struct texture; struct site : std::array { @@ -30,13 +38,24 @@ inline std::istream& operator >> (std::istream &i, site &x){ // Global variable for mom list struct tex_mom_list { size_t Nmoms; + +#if __HIP__ + hipTextureObject_t tex; +#else cudaTextureObject_t tex; +#endif void* devPtr; tex_mom_list() : Nmoms(0), tex(), devPtr(nullptr) {} +#ifdef __NVCC__ tex_mom_list(size_t Nmoms, cudaTextureObject_t tex, void* devPtr) : Nmoms(Nmoms), tex(tex), devPtr(devPtr) {} +#elif defined (__HIP__) + tex_mom_list(size_t Nmoms, hipTextureObject_t tex, void* devPtr) : + Nmoms(Nmoms), tex(tex), devPtr(devPtr) {} +#endif + inline __device__ int4 get(const size_t &i) const { #ifdef __NVCC__ @@ -63,18 +82,26 @@ struct pointer_holder { void copyToDeviceConstant() { if(devPointer != nullptr) { - cudaError_t err = cudaMemcpyToSymbol( *devPointer, hostPointer, bytes*size); - if (err != cudaSuccess) { +#ifdef __NVCC__ + cudaError_t err = cudaMemcpyToSymbol( *devPointer, hostPointer, bytes*size); + if (err != cudaSuccess) { errorQuda("Failed to copy constant host memory of size to device %zu \n", size); - } + } +#elif defined (__HIP__) + HIP_CHECK(hipMemcpy(*devPointer, hostPointer, bytes*size, hipMemcpyHostToDevice)); +#endif } } void copyFromDeviceConstant() { if(false and devPointer != nullptr) { +#ifdef __NVCC__ cudaError_t err = cudaMemcpyFromSymbol(hostPointer, *devPointer, bytes*size, 0, cudaMemcpyDeviceToHost); if (err != cudaSuccess) { errorQuda("Failed to copy constant host memory of size to device %zu \n", size); } +#else + qudaMemcpy(hostPointer, *devPointer, bytes*size,qudaMemcpyDeviceToHost); +#endif } } bool checkDeviceConstant() { diff --git a/include/global/PLEGMA_templates.h b/include/global/PLEGMA_templates.h index 60a8c9b0..4a91d67f 100644 --- a/include/global/PLEGMA_templates.h +++ b/include/global/PLEGMA_templates.h @@ -45,20 +45,22 @@ template inline void hostFree(T &ptr) { } // Pinned memory allocation -template inline void hostMallocPinned(T &ptr, size_t size){ - cudaError_t err = cudaMallocHost((void**)&ptr, size); - if (err != cudaSuccess) { - errorQuda("Failed to allocate host memory of size %zu \n", size); - } +template inline void hostMallocPinned(T &ptr, size_t size){ + //ptr=(T*)pinned_malloc(size); + //cudaError_t err = cudaMallocHost((void**)&ptr, size); + //if (err != cudaSuccess) { + // errorQuda("Failed to allocate host memory of size %zu \n", size); + //} HGC_used_memory += size; } template inline void hostFreePinned(T &ptr, size_t size) { - cudaError_t err = cudaFreeHost(ptr); - if (err != cudaSuccess) { - errorQuda("Failed to free host memory of size %zu \n", size); - } - ptr=NULL; + //cudaError_t err = cudaFreeHost(ptr); + //if (err != cudaSuccess) { + // errorQuda("Failed to free host memory of size %zu \n", size); + //} + host_free(ptr); + //ptr=NULL; HGC_used_memory -= size; } template inline void hostFreePinned(T &ptr) { diff --git a/include/plegma_define.h.in b/include/plegma_define.h.in new file mode 100644 index 00000000..8a376840 --- /dev/null +++ b/include/plegma_define.h.in @@ -0,0 +1,34 @@ +/** + @file quda_define.h + @brief Macros defined set by the cmake build system. This file + should not be edited manually. + */ + +/** + * @def __COMPUTE_CAPABILITY__ + * @brief This macro sets the target GPU architecture, which is + * defined on both host and device. + */ +#define __COMPUTE_CAPABILITY__ @COMP_CAP@0 + +/** + * @def QUDA_TARGET_CUDA + * @brief This macro is set by CMake if the CUDA Build Target is selected + */ +#cmakedefine QUDA_TARGET_CUDA @QUDA_TARGET_CUDA@ + +/** + * @def QUDA_TARGET_HIP + * @brief This macro is set by CMake if the HIP Build target is selected + */ +#cmakedefine QUDA_TARGET_HIP @QUDA_TARGET_HIP@ + +/** + * @def QUDA_TARGET_SYCL + * @brief This macro is set by CMake if the SYCL Build target is selected + */ +#cmakedefine QUDA_TARGET_SYCL @QUDA_TARGET_SYCL@ + +#if !defined(QUDA_TARGET_CUDA) && !defined(QUDA_TARGET_HIP) && !defined(QUDA_TARGET_SYCL) +#error "No QUDA_TARGET selected" +#endif diff --git a/include/target/cuda/PLEGMA_cuBLAS.h b/include/target/cuda/PLEGMA_cuBLAS.h new file mode 100644 index 00000000..50d1a584 --- /dev/null +++ b/include/target/cuda/PLEGMA_cuBLAS.h @@ -0,0 +1,180 @@ +#if defined(HAVE_MKL) && defined(HAVE_OPENBLAS) +#error Cannot define both mkl and openBLAS +#endif +#include +#if defined(HAVE_MKL) +#include +#elif defined(HAVE_OPENBLAS) +#include +//#include // do not know when is needed or not +#else +#error Neither mkl nor openBLAS have been defined +#endif + +#include +#include +#include +#include +#pragma once + +namespace cuBLAS{ + + template + inline void axpy(int NN, Float val[2], Float *x, Float *y){} + template<> + inline void axpy(int NN, float val[2], float *x, float *y){ + cuComplex cu_val = make_cuComplex(val[0],val[1]); + cublasStatus_t error = cublasCaxpy(HGC_cublas_handle,NN, &cu_val, (cuComplex*) x, 1, (cuComplex*) y, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCaxpy failed with error %d", error); + } + template<> + inline void axpy(int NN, double val[2], double *x, double *y){ + cuDoubleComplex cu_val = make_cuDoubleComplex(val[0],val[1]); + cublasStatus_t error = cublasZaxpy(HGC_cublas_handle, NN, &cu_val,(cuDoubleComplex*) x, 1, (cuDoubleComplex*) y, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZaxpy failed with error %d", error); + } + + //------------------------------------------------------------------ + template + inline void scal(int NN, const Float val, Float *x); + template<> + inline void scal(int NN, const float val, float *x){ + cublasStatus_t error = cublasCsscal(HGC_cublas_handle, NN, &val, (cuComplex*) x, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCsscal failed with error %d", error); + } + template<> + inline void scal(int NN, const double val, double *x){ + cublasStatus_t error = cublasZdscal(HGC_cublas_handle, NN, &val, (cuDoubleComplex*) x, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZdscal failed with error %d", error); + } + //------------------------------------------------------------------ + template + inline void cscal(int NN, const Float val[2], Float *x); + template<> + inline void cscal(int NN, const float val[2], float *x){ + cuComplex cu_val = make_cuComplex(val[0],val[1]); + cublasStatus_t error = cublasCscal(HGC_cublas_handle, NN, &cu_val, (cuComplex*) x, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCscal failed with error %d", error); + } + template<> + inline void cscal(int NN, const double val[2], double *x){ + cuDoubleComplex cu_val = make_cuDoubleComplex(val[0],val[1]); + cublasStatus_t error = cublasZscal(HGC_cublas_handle, NN, &cu_val, (cuDoubleComplex*) x, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZscal failed with error %d", error); + } + //----------------------------------------------------------------- + template + inline std::complex dot(int NN, const Float *x, const Float *y); + template<> + inline std::complex dot(int NN, const float *x, const float *y){ + cuComplex cu_res; + cublasStatus_t error = cublasCdotc(HGC_cublas_handle, NN,(cuComplex*)x, 1, (cuComplex*)y,1,&cu_res); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); + return std::complex(cu_res.x,cu_res.y); + } + template<> + inline std::complex dot(int NN, const double *x, const double *y){ + cuDoubleComplex cu_res; + cublasStatus_t error = cublasZdotc(HGC_cublas_handle, NN,(cuDoubleComplex*)x, 1, (cuDoubleComplex*)y,1,&cu_res); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZdotc failed with error %d", error); + return std::complex(cu_res.x,cu_res.y); + } + template + inline std::complex dot(int NN, const Float *x, const Float *y, MPI_Comm comm) { + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + std::complex result, res = cuBLAS::dot(NN, x, y); + int mpiErr = MPI_Allreduce((Float*) &res, (Float*) &result, 2, + MPI_Type(), MPI_SUM, comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + return result; + } + //----------------------------------------------------------------- + template + inline Float norm(int NN, const Float *x); + template<> + inline float norm(int NN, const float *x){ + float res; + cublasStatus_t error = cublasScnrm2(HGC_cublas_handle, NN, (cuComplex*)x, 1, &res); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); + return res; + } + template<> + inline double norm(int NN, const double *x){ + double res; + cublasStatus_t error = cublasDznrm2(HGC_cublas_handle, NN, (cuDoubleComplex*)x, 1, &res); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCdotc failed with error %d", error); + return res; + } + template + inline Float norm(int NN, const Float *x, MPI_Comm comm) { + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + Float result, loc_res = std::pow(cuBLAS::norm(NN, x),2); + int mpiErr = MPI_Allreduce(&loc_res, &result, 1, MPI_Type(), MPI_SUM, + comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + return sqrt(result); + } + + + //--------------------------------------------------------- + template + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){} + + template<> + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, float alpha[2], float* A, float* x, float beta[2], float* y){ + cublasOperation_t Oper; + cuComplex cu_alpha = make_cuComplex(alpha[0],alpha[1]); + cuComplex cu_beta = make_cuComplex(beta[0],beta[1]); + switch(trans){case(NOTRANS): Oper=CUBLAS_OP_N; break; case(TRANS): Oper=CUBLAS_OP_T; break; case(DAGGER): Oper=CUBLAS_OP_C; break;} + cublasStatus_t error = cublasCgemv(HGC_cublas_handle, Oper, m, n, &cu_alpha, (cuComplex*) A, m, (cuComplex*) x, 1, &cu_beta, (cuComplex*) y, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCgemv failed with error %d", error); + } + + template<> + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, double alpha[2], double* A, double* x, double beta[2], double* y){ + cublasOperation_t Oper; + cuDoubleComplex cu_alpha = make_cuDoubleComplex(alpha[0],alpha[1]); + cuDoubleComplex cu_beta = make_cuDoubleComplex(beta[0],beta[1]); + switch(trans){case(NOTRANS): Oper=CUBLAS_OP_N; break; case(TRANS): Oper=CUBLAS_OP_T; break; case(DAGGER): Oper=CUBLAS_OP_C; break;} + cublasStatus_t error = cublasZgemv(HGC_cublas_handle, Oper, m, n, &cu_alpha, (cuDoubleComplex*) A, m,(cuDoubleComplex*) x, 1, &cu_beta,(cuDoubleComplex*) y, 1); + if(error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasZgemv failed with error %d", error); + } + + // In case of m is partitioned and trans=NOTRANS OR m is not partitioned one can use whatever trans + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){ + gemv_(trans, m, n, alpha, A, x, beta, y); + } + // In case that m is partitioned and we take trans of dagger then reduction is needed + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, Float *yHost, MPI_Comm comm){ + if(trans == NOTRANS) PLEGMA_error("Use gemv without MPI comm"); + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + cuBLAS::gemv_(trans, m, n, alpha, A, x, beta, y); + qudaMemcpy(yHost,y,n*2*sizeof(Float),qudaMemcpyDeviceToHost); + checkQudaError(); + int mpiErr = MPI_Allreduce(MPI_IN_PLACE,yHost,n*2,MPI_Type(yHost),MPI_SUM,comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + } + + // In case that m is partitioned and we take trans of dagger then reduction is needed + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, MPI_Comm comm){ + Float yHost[n*2]; + cuBLAS::gemv(trans,m, n, alpha, A, x, beta, y, yHost,comm); + qudaMemcpy(y,yHost,sizeof(yHost),qudaMemcpyHostToDevice); + checkQudaError(); + } +} +//=================================================================// + +namespace plegma{ + template + void elemWiseMul(int NN, Float* x, Float* y); + + template + void axpbypcz(int NN, Float a[2], Float* x, Float b[2], Float* y, Float c[2], Float* z); +} diff --git a/include/target/hip/PLEGMA_hipBLAS.h b/include/target/hip/PLEGMA_hipBLAS.h new file mode 100644 index 00000000..1120795c --- /dev/null +++ b/include/target/hip/PLEGMA_hipBLAS.h @@ -0,0 +1,157 @@ +#include +#include +#pragma once +namespace cuBLAS{ + + template + inline void axpy(int NN, Float val[2], Float *x, Float *y){} + template<> + inline void axpy(int NN, float val[2], float *x, float *y){ + hipblasComplex cu_val(val[0],val[1]); + hipblasStatus_t error = hipblasCaxpy(HGC_hipblas_handle,NN, &cu_val, (hipblasComplex*) x, 1, (hipblasComplex*) y, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCaxpy failed with error %d", error); + } + template<> + inline void axpy(int NN, double val[2], double *x, double *y){ + hipblasDoubleComplex cu_val(val[0],val[1]); + hipblasStatus_t error = hipblasZaxpy(HGC_hipblas_handle, NN, &cu_val,(hipblasDoubleComplex*) x, 1, (hipblasDoubleComplex*) y, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasZaxpy failed with error %d", error); + } + + //------------------------------------------------------------------ + template + inline void scal(int NN, const Float val, Float *x); + template<> + inline void scal(int NN, const float val, float *x){ + hipblasStatus_t error = hipblasCsscal(HGC_hipblas_handle, NN, &val, (hipblasComplex*) x, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCsscal failed with error %d", error); + } + template<> + inline void scal(int NN, const double val, double *x){ + hipblasStatus_t error = hipblasZdscal(HGC_hipblas_handle, NN, &val, (hipblasDoubleComplex*) x, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasZdscal failed with error %d", error); + } + //------------------------------------------------------------------ + template + inline void cscal(int NN, const Float val[2], Float *x); + template<> + inline void cscal(int NN, const float val[2], float *x){ + hipblasComplex cu_val(val[0],val[1]); + hipblasStatus_t error = hipblasCscal(HGC_hipblas_handle, NN, &cu_val, (hipblasComplex*) x, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCscal failed with error %d", error); + } + template<> + inline void cscal(int NN, const double val[2], double *x){ + hipblasDoubleComplex cu_val(val[0],val[1]); + hipblasStatus_t error = hipblasZscal(HGC_hipblas_handle, NN, &cu_val, (hipblasDoubleComplex*) x, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasZscal failed with error %d", error); + } + //----------------------------------------------------------------- + template + inline std::complex dot(int NN, const Float *x, const Float *y); + template<> + inline std::complex dot(int NN, const float *x, const float *y){ + hipblasComplex cu_res; + hipblasStatus_t error = hipblasCdotc(HGC_hipblas_handle, NN,(hipblasComplex*)x, 1, (hipblasComplex*)y,1,&cu_res); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCdotc failed with error %d", error); + return std::complex(cu_res.real(),cu_res.imag()); + } + template<> + inline std::complex dot(int NN, const double *x, const double *y){ + hipblasDoubleComplex cu_res; + hipblasStatus_t error = hipblasZdotc(HGC_hipblas_handle, NN,(hipblasDoubleComplex*)x, 1, (hipblasDoubleComplex*)y,1,&cu_res); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasZdotc failed with error %d", error); + return std::complex(cu_res.real(),cu_res.imag()); + } + template + inline std::complex dot(int NN, const Float *x, const Float *y, MPI_Comm comm) { + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + std::complex result, res = cuBLAS::dot(NN, x, y); + int mpiErr = MPI_Allreduce((Float*) &res, (Float*) &result, 2, + MPI_Type(), MPI_SUM, comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + return result; + } + //----------------------------------------------------------------- + template + inline Float norm(int NN, const Float *x); + template<> + inline float norm(int NN, const float *x){ + float res; + hipblasStatus_t error = hipblasScnrm2(HGC_hipblas_handle, NN, (hipblasComplex*)x, 1, &res); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCdotc failed with error %d", error); + return res; + } + template<> + inline double norm(int NN, const double *x){ + double res; + hipblasStatus_t error = hipblasDznrm2(HGC_hipblas_handle, NN, (hipblasDoubleComplex*)x, 1, &res); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCdotc failed with error %d", error); + return res; + } + template + inline Float norm(int NN, const Float *x, MPI_Comm comm) { + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + Float result, loc_res = std::pow(cuBLAS::norm(NN, x),2); + int mpiErr = MPI_Allreduce(&loc_res, &result, 1, MPI_Type(), MPI_SUM, + comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + return sqrt(result); + } + + + //--------------------------------------------------------- + template + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){} + + template<> + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, float alpha[2], float* A, float* x, float beta[2], float* y){ + hipblasOperation_t Oper; + hipblasComplex cu_alpha(alpha[0],alpha[1]); + hipblasComplex cu_beta(beta[0],beta[1]); + switch(trans){case(NOTRANS): Oper=HIPBLAS_OP_N; break; case(TRANS): Oper=HIPBLAS_OP_T; break; case(DAGGER): Oper=HIPBLAS_OP_C; break;} + hipblasStatus_t error = hipblasCgemv(HGC_hipblas_handle, Oper, m, n, &cu_alpha, (hipblasComplex*) A, m, (hipblasComplex*) x, 1, &cu_beta, (hipblasComplex*) y, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasCgemv failed with error %d", error); + } + + template<> + inline void gemv_(OPER_MATR_BLAS trans, int m, int n, double alpha[2], double* A, double* x, double beta[2], double* y){ + hipblasOperation_t Oper; + hipblasDoubleComplex cu_alpha(alpha[0],alpha[1]); + hipblasDoubleComplex cu_beta(beta[0],beta[1]); + switch(trans){case(NOTRANS): Oper=HIPBLAS_OP_N; break; case(TRANS): Oper=HIPBLAS_OP_T; break; case(DAGGER): Oper=HIPBLAS_OP_C; break;} + hipblasStatus_t error = hipblasZgemv(HGC_hipblas_handle, Oper, m, n, &cu_alpha, (hipblasDoubleComplex*) A, m,(hipblasDoubleComplex*) x, 1, &cu_beta,(hipblasDoubleComplex*) y, 1); + if(error != HIPBLAS_STATUS_SUCCESS) PLEGMA_error("hipblasZgemv failed with error %d", error); + } + + // In case of m is partitioned and trans=NOTRANS OR m is not partitioned one can use whatever trans + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y){ + gemv_(trans, m, n, alpha, A, x, beta, y); + } + // In case that m is partitioned and we take trans of dagger then reduction is needed + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, Float *yHost, MPI_Comm comm){ + if(trans == NOTRANS) PLEGMA_error("Use gemv without MPI comm"); + if(comm == MPI_COMM_NULL) PLEGMA_error("Communicator is NULL and cannot be used for MPI reduction"); + cuBLAS::gemv_(trans, m, n, alpha, A, x, beta, y); + qudaMemcpy(yHost,y,n*2*sizeof(Float),qudaMemcpyDeviceToHost); + checkQudaError(); + int mpiErr = MPI_Allreduce(MPI_IN_PLACE,yHost,n*2,MPI_Type(yHost),MPI_SUM,comm); + if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Allreduce failed with error %d\n", mpiErr); + } + + // In case that m is partitioned and we take trans of dagger then reduction is needed + // Cannot work if n is partitioned + template + inline void gemv(OPER_MATR_BLAS trans, int m, int n, Float alpha[2], Float* A, Float* x, Float beta[2], Float* y, MPI_Comm comm){ + Float yHost[n*2]; + cuBLAS::gemv(trans,m, n, alpha, A, x, beta, y, yHost,comm); + qudaMemcpy(y,yHost,sizeof(yHost),qudaMemcpyHostToDevice); + checkQudaError(); + } +} +//=================================================================// + diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index ba3b9da5..05b39c05 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,5 +1,22 @@ +# cmake-format: off + +# this allows simplified running of clang-tid +if(${CMAKE_BUILD_TYPE} STREQUAL "DEVEL") + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +endif() + +#build up git version add -debug to GITVERSION if we build with debug options enabled +string(REGEX MATCH [Dd][Ee][Bb][Uu][Gg] DEBUG_BUILD ${CMAKE_BUILD_TYPE}) +if(DEBUG_BUILD) + if(GITVERSION) + set(GITVERSION ${GITVERSION}-debug) + else() + set(GITVERSION debug) + endif() +endif() + set (PLEGMA_OBJS - PLEGMA_Correlator.cu + PLEGMA_Correlator.cu PLEGMA.cu PLEGMA_Propagator.cu PLEGMA_Vector.cu @@ -7,6 +24,8 @@ set (PLEGMA_OBJS PLEGMA_Field.cu PLEGMA_Su3field.cu PLEGMA_Random.cu + kernels/PLEGMA_seqSourceNucleon.cu + PLEGMA_BLAS.cu kernels/PLEGMA_baryons_NtoN.cu kernels/PLEGMA_seqSourceNucleon.cu kernels/PLEGMA_threep_local.cu @@ -19,12 +38,12 @@ set (PLEGMA_OBJS kernels/PLEGMA_threep_threeD_part4.cu kernels/PLEGMA_threep_wilsonLine.cu kernels/PLEGMA_threep_staple.cu - kernels/PLEGMA_threep_qgq.cu + kernels/PLEGMA_threep_qgq.cu PLEGMA_QLoops.cu PLEGMA_FT.cu PLEGMA_BLAS.cu PLEGMA_Fmunu.cu - PLEGMA_U1Gauge.cu + PLEGMA_U1Gauge.cu ) if(PLEGMA_UDSC_BARYONS) LIST(APPEND PLEGMA_OBJS @@ -63,130 +82,190 @@ endif() FOREACH(item ${PLEGMA_OBJS}) STRING(REGEX MATCH ".+\\.cu$" item_match ${item}) IF(item_match) - LIST(APPEND QUDA_CU_OBJS ${item}) + LIST(APPEND PLEGMA_CU_OBJS ${item}) ENDIF(item_match) ENDFOREACH(item ${PLEGMA_OBJS}) -LIST(REMOVE_ITEM PLEGMA_OBJS ${QUDA_CU_OBJS}) +LIST(REMOVE_ITEM PLEGMA_OBJS ${PLEGMA_CU_OBJS}) +message(STATUS "PLEGMA_OBJS=${PLEGMA_OBJS}") +message(STATUS "PLEGMA_CU_OBJS=${PLEGMA_CU_OBJS}") -# QUDA_CU_OBJS shoudl contain all cuda files now -# PLEGMA_OBJS all c, cpp, fortran sources -# include_directories -include_directories(${QUDA_HOME}/lib/) -#PLEGMA: -include_directories(code_pieces) -include_directories(SYSTEM ${PLEGMA_GSLHOME}/include) -include_directories(SYSTEM ${PLEGMA_HDF5HOME}/include) +# PLEGMA_CU_OBJS should contain all cuda files now PLEGMA_OBJS all c, cpp, fortran sources +# if we have a git version make version.cpp depend on git head so that it is rebuild if the git sha changed -include_directories(SYSTEM ${PLEGMA_HDF5HOME}/src) -include_directories(SYSTEM ${PLEGMA_LIMEHOME}/include) -if(PLEGMA_ARPACK) - include_directories(SYSTEM ${PLEGMA_ARPACKHOME}/PARPACK/SRC/MPI) - include_directories(SYSTEM ${PLEGMA_ARPACKHOME}/PARPACK/UTIL/MPI) - include_directories(SYSTEM ${PLEGMA_ARPACKHOME}/SRC) - include_directories(SYSTEM ${PLEGMA_ARPACKHOME}/UTIL) -endif(PLEGMA_ARPACK) -#PLEGMA: - -set(PLEGMA_LIB ${QUDA_CU_OBJS}) - -# generate a cmake object library for all cpp files first if(PLEGMA_OBJS) - add_library(PLEGMA_cpp OBJECT ${PLEGMA_OBJS}) + add_library(plegma_cpp OBJECT ${PLEGMA_OBJS}) - # add some definitions that cause issues with cmake 3.7 and nvcc only to cpp files - target_compile_definitions(PLEGMA_cpp PUBLIC -DQUDA_HASH="${HASH}") - if(PLEGMA_BUILD_SHAREDLIB) - set_target_properties(PLEGMA_cpp PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + target_compile_definitions(plegma_cpp PUBLIC -DPLEGMA_HASH="${HASH}") + if(GITVERSION) + target_compile_definitions(plegma_cpp PUBLIC -DGITVERSION="${GITVERSION}") + endif() + set_target_properties(plegma_cpp PROPERTIES POSITION_INDEPENDENT_CODE TRUE) +endif() +if(${CMAKE_BUILD_TYPE} STREQUAL "DEVEL") + if(GITVERSION) + find_path( + PLEGMA_GITDIR NAME HEAD + PATHS ${CMAKE_SOURCE_DIR}/.git/logs + NO_DEFAULT_PATH) + include(AddFileDependencies) + if(PLEGMA_GITDIR) + add_file_dependencies(version.cpp ${PLEGMA_GITDIR}/HEAD) + endif() endif() + mark_as_advanced(PLEGMA_GITDIR) +endif() + +# QUDA_CU_OBJS shoudl contain all cuda files now +# PLEGMA_OBJS all c, cpp, fortran sources - LIST(APPEND PLEGMA_LIB $) +# add target specific files +if(${PLEGMA_TARGET_TYPE} STREQUAL "CUDA") + enable_language(CUDA) + find_package(CUDAWrapper) + if(PLEGMA_BUILD_SHAREDLIB) + cuda_add_library(plegma SHARED ${PLEGMA_LIB}) + else() + cuda_add_library(plegma STATIC ${PLEGMA_LIB}) + endif() + include(targets/cuda/target_cuda.cmake) +endif() +if(${PLEGMA_TARGET_TYPE} STREQUAL "HIP") + include(targets/hip/target_hip.cmake) +endif() +if(${PLEGMA_TARGET_TYPE} STREQUAL "SYCL") + include(targets/sycl/target_sycl.cmake) endif() # make one library -if(PLEGMA_BUILD_SHAREDLIB) - cuda_add_library(PLEGMA SHARED ${PLEGMA_LIB}) -else() - cuda_add_library(PLEGMA STATIC ${PLEGMA_LIB}) -endif() -target_include_directories(PLEGMA PUBLIC $ - $) +target_sources(plegma PRIVATE ${PLEGMA_CU_OBJS}) +# +# ${PLEGMA_CU_OBJS}) + +include(CheckLinkerFlag) +check_linker_flag(CXX "-Wl,--compress-debug-sections=zlib" PLEGMA_LINKER_COMPRESS) +target_link_options(plegma PRIVATE $<$:$<${PLEGMA_LINKER_COMPRESS}:-Wl,--compress-debug-sections=zlib>>) + + +# set up QUDA compile options, put them before target specific options +target_compile_options( + plegma BEFORE PRIVATE $<$: + $,-w,-Wall -Wextra> + $<$:-Werror> + >) + +target_compile_definitions( + plegma PRIVATE $<$:DEVEL> $<$:HOST_DEBUG> $<$:HOST_DEBUG> + $<$:DEVICE_DEBUG> $<$:HOST_DEBUG>) +target_include_directories(plegma PRIVATE .) +target_include_directories(plegma PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + + +# include_directories +target_include_directories(plegma SYSTEM PRIVATE ${QUDA_HOME}/lib/) +#PLEGMA: +target_include_directories(plegma SYSTEM PRIVATE ${PLEGMA_GSLHOME}/include) +target_include_directories(plegma SYSTEM PRIVATE ${PLEGMA_HDF5HOME}/include) + +target_include_directories(plegma SYSTEM PRIVATE ${PLEGMA_HDF5HOME}/src) +target_include_directories(plegma SYSTEM PRIVATE ${PLEGMA_LIMEHOME}/include) +if(PLEGMA_ARPACK) + target_include_directories(SYSTEM PRIVATE ${PLEGMA_ARPACKHOME}/PARPACK/SRC/MPI) + target_include_directories(SYSTEM PRIVATE ${PLEGMA_ARPACKHOME}/PARPACK/UTIL/MPI) + target_include_directories(SYSTEM PRIVATE ${PLEGMA_ARPACKHOME}/SRC) + target_include_directories(SYSTEM PRIVATE ${PLEGMA_ARPACKHOME}/UTIL) +endif(PLEGMA_ARPACK) -LIST(APPEND PLEGMA ${PLEGMA_GSL_LIB}) -LIST(APPEND PLEGMA ${PLEGMA_HDF5_LIB}) -LIST(APPEND PLEGMA ${PLEGMA_LIME_LIB}) + +LIST(APPEND plegma ${PLEGMA_GSL_LIB}) +LIST(APPEND plegma ${PLEGMA_HDF5_LIB}) +LIST(APPEND plegma ${PLEGMA_LIME_LIB}) if(PLEGMA_ARPACK) - LIST(APPEND PLEGMA ${PLEGMA_PARPACK_LIB}) - LIST(APPEND PLEGMA ${PLEGMA_ARPACK_LIB}) + LIST(APPEND plegma ${PLEGMA_PARPACK_LIB}) + LIST(APPEND plegma ${PLEGMA_ARPACK_LIB}) endif(PLEGMA_ARPACK) if(PLEGMA_OPENBLAS) - LIST(APPEND PLEGMA ${PLEGMA_OPENBLAS_GOMP}) - LIST(APPEND PLEGMA ${PLEGMA_OPENBLAS_LIB}) + LIST(APPEND plegma ${PLEGMA_OPENBLAS_GOMP}) + LIST(APPEND plegma ${PLEGMA_OPENBLAS_LIB}) endif(PLEGMA_OPENBLAS) if(PLEGMA_MKL) - LIST(APPEND PLEGMA ${PLEGMA_MKL_LIB_IOMP5}) - LIST(APPEND PLEGMA ${PLEGMA_MKL_LIB_GFORTRAN}) - LIST(APPEND PLEGMA ${PLEGMA_MKL_LIB_LP64}) - LIST(APPEND PLEGMA ${PLEGMA_MKL_LIB_THREAD}) - LIST(APPEND PLEGMA ${PLEGMA_MKL_LIB_CORE}) + LIST(APPEND plegma ${PLEGMA_MKL_LIB_IOMP5}) + LIST(APPEND plegma ${PLEGMA_MKL_LIB_GFORTRAN}) + LIST(APPEND plegma ${PLEGMA_MKL_LIB_LP64}) + LIST(APPEND plegma ${PLEGMA_MKL_LIB_THREAD}) + LIST(APPEND plegma ${PLEGMA_MKL_LIB_CORE}) endif(PLEGMA_MKL) # add dependencies for linking # if CUDA_cuda_LIBRARY (driver api) is not found look for stubs -FIND_LIBRARY(CUDA_cuda_LIBRARY cuda HINTS ${CUDA_TOOLKIT_ROOT_DIR}/lib/ ${CUDA_TOOLKIT_ROOT_DIR}/lib/stubs) - -target_link_libraries(PLEGMA ${CMAKE_THREAD_LIBS_INIT} ${QUDA_LIB} ${QUDA_LIBS} ${CUDA_cuda_LIBRARY}) -add_dependencies(PLEGMA BoostPP) +#FIND_LIBRARY(CUDA_cuda_LIBRARY cuda HINTS ${CUDA_TOOLKIT_ROOT_DIR}/lib/ ${CUDA_TOOLKIT_ROOT_DIR}/lib/stubs) +if(${PLEGMA_TARGET_TYPE} STREQUAL "CUDA") + target_link_libraries(plegma ${CMAKE_THREAD_LIBS_INIT} ${QUDA_LIB} ${CUDA_cuda_LIBRARY}) +endif() +if(${PLEGMA_TARGET_TYPE} STREQUAL "HIP") + target_link_libraries(plegma ${CMAKE_THREAD_LIBS_INIT} ${QUDA_LIB} ) # ${CUDA_cuda_LIBRARY}) +endif() +add_dependencies(plegma BoostPP) if(QUDA_QMP) - target_link_libraries(PLEGMA ${QMP_LIB} ${MPI_CXX_LIBRARIES}) + target_link_libraries(plegma ${QMP_LIB} ${MPI_CXX_LIBRARIES}) endif() if(QUDA_MPI) - target_link_libraries(PLEGMA ${MPI_CXX_LIBRARIES}) + target_link_libraries(plegma ${MPI_CXX_LIBRARIES}) endif() if(QUDA_MAGMA) - target_link_libraries(PLEGMA ${MAGMA}) + target_link_libraries(plegma ${MAGMA}) endif() if(QUDA_ARPACK) - target_link_libraries(PLEGMA ${PARPACK_OR_ARPACK}) + target_link_libraries(plegma ${PARPACK_OR_ARPACK}) endif() -# malloc.cpp uses both the driver and runtime api -# So we need to find the CUDA_cuda_LIBRARY (driver api) or the stub version -find_library(CUDA_cuda_LIBRARY cuda HINTS ${CUDA_TOOLKIT_ROOT_DIR}/lib/ ${CUDA_TOOLKIT_ROOT_DIR}/lib/stubs) -target_link_libraries(PLEGMA ${CUDA_cuda_LIBRARY}) +configure_file(../include/plegma_define.h.in ../include/plegma_define.h @ONLY) +install(FILES "${CMAKE_BINARY_DIR}/include/plegma_define.h" DESTINATION include/) + # until we define an install step copy the include directory to the build directory -ADD_CUSTOM_COMMAND(TARGET PLEGMA POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/include) +ADD_CUSTOM_COMMAND(TARGET plegma POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/include) # some hackery to prevent having old shared / static builds of quda messing with the current build -ADD_CUSTOM_COMMAND(TARGET PLEGMA PRE_LINK COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/libPLEGMA.a ${CMAKE_CURRENT_BINARY_DIR}/libPLEGMA.so) +ADD_CUSTOM_COMMAND(TARGET plegma PRE_LINK COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/libplegma.a ${CMAKE_CURRENT_BINARY_DIR}/libplegma.so) -install(TARGETS PLEGMA EXPORT PLEGMATargets +install(TARGETS plegma + EXPORT PLEGMATargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib - INCLUDES DESTINATION include) + INCLUDES + DESTINATION include) + +install( + EXPORT PLEGMATargets + FILE PLEGMATargets.cmake + NAMESPACE PLEGMA:: + DESTINATION lib/cmake/PLEGMA) install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ DESTINATION include) include(CMakePackageConfigHelpers) write_basic_package_version_file( - "${CMAKE_CURRENT_BINARY_DIR}/qudaConfigVersion.cmake" - VERSION ${QUDA_VERSION} + "${CMAKE_CURRENT_BINARY_DIR}/plegmaConfigVersion.cmake" + VERSION ${PLEGMA_VERSION} COMPATIBILITY AnyNewerVersion ) -export(EXPORT PLEGMATargets FILE "${CMAKE_CURRENT_BINARY_DIR}/PLEGMATargets.cmake" NAMESPACE quda::) -set(ConfigPackageLocation lib/cmake/quda/) -install(EXPORT PLEGMATargets NAMESPACE quda:: DESTINATION ${ConfigPackageLocation}) +include(CMakePackageConfigHelpers) + +configure_package_config_file(${CMAKE_SOURCE_DIR}/PLEGMAConfig.cmake.in PLEGMAConfig.cmake INSTALL_DESTINATION lib/cmake/PLEGMA) +#export(EXPORT PLEGMATargets FILE "${CMAKE_CURRENT_BINARY_DIR}/PLEGMATargets.cmake" NAMESPACE quda::) +#set(ConfigPackageLocation lib/cmake/quda/) +#install(EXPORT PLEGMATargets NAMESPACE quda:: DESTINATION ${ConfigPackageLocation}) add_custom_target(gen ${PYTHON_EXECUTABLE} ${PLEGMA_QUDA_HOME}/lib/generate/gen.py @@ -196,3 +275,4 @@ add_custom_target(gen ${PYTHON_EXECUTABLE} ${PLEGMA_QUDA_HOME}/lib/generate/gen. add_custom_target(mpi_nvtx ${PYTHON_EXECUTABLE} ${PLEGMA_QUDA_HOME}/lib/generate/wrap.py -g -o ${PLEGMA_QUDA_HOME}/lib/nvtx_pmpi.c ${PLEGMA_QUDA_HOME}/lib/generate/nvtx.w WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating mpi_nvtx wrapper" ) + diff --git a/lib/PLEGMA.cu b/lib/PLEGMA.cu index 7616882f..cd933b59 100644 --- a/lib/PLEGMA.cu +++ b/lib/PLEGMA.cu @@ -202,8 +202,8 @@ void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){ MPI_Comm_rank(HGC_timeComm,&tmp); assert(tmp==HGC_timeRank); - cublasStatus_t error = cublasCreate(&HGC_cublas_handle); - if (error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCreate failed with error %d", error); + //cublasStatus_t error = cublasCreate(&HGC_cublas_handle); + //if (error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("cublasCreate failed with error %d", error); HGC_init_PLEGMA_flag = true; PLEGMA_printf("PLEGMA has been initialized\n"); @@ -229,8 +229,8 @@ void plegma::PLEGMA_status(){ void plegma::PLEGMA_end() { // TODO: here we should destroy everything is created in init. - cublasStatus_t error = cublasDestroy(HGC_cublas_handle); - if (error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("\nError indestroying cublas context, error code = %d\n", error); + //cublasStatus_t error = cublasDestroy(HGC_cublas_handle); + //if (error != CUBLAS_STATUS_SUCCESS) PLEGMA_error("\nError indestroying cublas context, error code = %d\n", error); if(HDF5::isWriting()) { PLEGMA_printf("Waiting for HDF5 to finish the writing\n"); while(HDF5::isWriting()) sleep(0.001); diff --git a/lib/PLEGMA_BLAS.cu b/lib/PLEGMA_BLAS.cu index 2c67e548..f72966e5 100644 --- a/lib/PLEGMA_BLAS.cu +++ b/lib/PLEGMA_BLAS.cu @@ -1,4 +1,4 @@ -#include +#include #include namespace plegma{ diff --git a/lib/PLEGMA_Correlator.cu b/lib/PLEGMA_Correlator.cu index 47b2bd72..3c6a32c8 100644 --- a/lib/PLEGMA_Correlator.cu +++ b/lib/PLEGMA_Correlator.cu @@ -3,13 +3,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #ifdef PLEGMA_UDSC_BARYONS #include @@ -1085,8 +1085,8 @@ writeHDF5(std::string filename) const { } -template class PLEGMA_Correlator; -template class PLEGMA_Correlator; +template class plegma::PLEGMA_Correlator; +template class plegma::PLEGMA_Correlator; diff --git a/lib/PLEGMA_FT.cu b/lib/PLEGMA_FT.cu index 13e743a5..9e9e6bcb 100644 --- a/lib/PLEGMA_FT.cu +++ b/lib/PLEGMA_FT.cu @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -15,34 +15,73 @@ using namespace plegma; template PLEGMA_FT::PLEGMA_FT(int Q2_max, int D3D4, bool accum, int dimT): Q2_max(Q2_max), dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ - if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); + if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions %d\n",dims); if(Q2_max < 0) PLEGMA_error("The maximum number of Q2 cannot be negative\n"); if(dimT<0 || dimT>HGC_localL[DIM_T]) PLEGMA_error("The time dimension cannot be negative or larger than local size\n"); createMom(); } template -template -PLEGMA_FT::PLEGMA_FT(std::vector mom, int D3D4, bool accum, int dimT): +PLEGMA_FT::PLEGMA_FT(std::vector mom, int D3D4, bool accum, int dimT): dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); - if(mom.size() != dims) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT"); + if(mom.size() != dims) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d\n",mom.size(), dims); VFloat momF(mom.begin(),mom.end()); momList.push_back(momF); } template -template -PLEGMA_FT::PLEGMA_FT( std::vector> &moms, int D3D4, bool accum, int dimT ): +PLEGMA_FT::PLEGMA_FT( std::vector> &moms, int D3D4, bool accum, int dimT ): + dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ + if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); + for(auto &mom : moms){ + if(mom.size()%dims != 0) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d",mom.size(),dims); + VFloat momF(mom.begin(),mom.end()); + momList.push_back(momF); + } +} + +template +PLEGMA_FT::PLEGMA_FT(std::vector mom, int D3D4, bool accum, int dimT): + dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ + if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); + if(mom.size() != dims) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d\n",mom.size(), dims); + VFloat momF(mom.begin(),mom.end()); + momList.push_back(momF); +} + +template +PLEGMA_FT::PLEGMA_FT( std::vector> &moms, int D3D4, bool accum, int dimT ): + dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ + if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); + for(auto &mom : moms){ + if(mom.size()%dims != 0) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d",mom.size(),dims); + VFloat momF(mom.begin(),mom.end()); + momList.push_back(momF); + } +} + +template +PLEGMA_FT::PLEGMA_FT(std::vector mom, int D3D4, bool accum, int dimT): + dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ + if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); + if(mom.size() != dims) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d\n",mom.size(), dims); + VFloat momF(mom.begin(),mom.end()); + momList.push_back(momF); +} + +template +PLEGMA_FT::PLEGMA_FT( std::vector> &moms, int D3D4, bool accum, int dimT ): dof(0), h_elem(nullptr), sizeN(0), dims(D3D4), dimT(D3D4==3?dimT:1), accum(accum){ if(dims!= 3 && dims !=4) PLEGMA_error("This class transforms only 3 and 4 dimensions\n"); for(auto &mom : moms){ - if(mom.size()%dims != 0) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %d %d",mom.size(),dims); + if(mom.size()%dims != 0) PLEGMA_error("The size of the momentum vector does not match the dimensionality of FT %zu %d",mom.size(),dims); VFloat momF(mom.begin(),mom.end()); momList.push_back(momF); } } + template void PLEGMA_FT::zero(){ if(h_elem) memset(h_elem.get(), 0, Nmoms()*dimT*dof*2*sizeof(Float)); @@ -82,17 +121,31 @@ void PLEGMA_FT::checkAllocation(int newDof){ template std::shared_ptr PLEGMA_FT::getTexMomList() { +#ifdef __NVCC__ cudaChannelFormatDesc desc; memset(&desc, 0, sizeof(cudaChannelFormatDesc)); desc.f = cudaChannelFormatKindSigned; +#elif defined (__HIP__) + hipChannelFormatDesc desc; + memset(&desc, 0, sizeof(hipChannelFormatDesc)); + desc.f = hipChannelFormatKindSigned; +#endif desc.x = 8*4; desc.y = 8*4; desc.z = 8*4; desc.w = 8*4; - + +#ifdef __NVCC__ cudaResourceDesc resDesc; +#elif defined (__HIP__) + hipResourceDesc resDesc; +#endif memset(&resDesc, 0, sizeof(resDesc)); +#ifdef __NVCC__ resDesc.resType = cudaResourceTypeLinear; +#elif defined (__HIP__) + resDesc.resType = hipResourceTypeLinear; +#endif resDesc.res.linear.desc = desc; void * devPtr; @@ -107,10 +160,11 @@ std::shared_ptr PLEGMA_FT::getTexMomList() { hostPtr[i*N_DIMS+j]=(int) std::lround(momList[i][j]); } } - cudaMemcpy(devPtr, hostPtr, sizeof(hostPtr), cudaMemcpyHostToDevice ); + qudaMemcpy(devPtr, hostPtr, sizeof(hostPtr), qudaMemcpyHostToDevice ); resDesc.res.linear.devPtr = devPtr; resDesc.res.linear.sizeInBytes = sizeof(hostPtr); +#if defined __NVCC__ cudaTextureDesc texDesc; memset(&texDesc, 0, sizeof(texDesc)); texDesc.readMode = cudaReadModeElementType; @@ -119,6 +173,16 @@ std::shared_ptr PLEGMA_FT::getTexMomList() { cudaCreateTextureObject(&tex, &resDesc, &texDesc, NULL); return std::shared_ptr(new tex_mom_list(Nmoms(), tex, devPtr), [](tex_mom_list* moms) { cudaDestroyTextureObject(moms->tex); device_free(moms->devPtr);}); +#elif defined (__HIP__) + hipTextureDesc texDesc; + memset(&texDesc, 0, sizeof(texDesc)); + texDesc.readMode = hipReadModeElementType; + + hipTextureObject_t tex; + hipCreateTextureObject(&tex, &resDesc, &texDesc, NULL); + + return std::shared_ptr(new tex_mom_list(Nmoms(), tex, devPtr), [](tex_mom_list* moms) { hipDestroyTextureObject(moms->tex); device_free(moms->devPtr);}); +#endif } template @@ -380,14 +444,15 @@ writeHDF5(std::string filename, int timeshift, bool append) const{ writer.write_dataset("mvec", mvec, momShape); } -template class PLEGMA_FT; -template class PLEGMA_FT; - -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); -template PLEGMA_FT::PLEGMA_FT( std::vector> &,int,bool,int); -template PLEGMA_FT::PLEGMA_FT( std::vector> &,int,bool,int); +template class plegma::PLEGMA_FT; +template class plegma::PLEGMA_FT; +#if 0 +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT(std::vector,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT( std::vector> &,int,bool,int); +template plegma::PLEGMA_FT::PLEGMA_FT( std::vector> &,int,bool,int); +#endif diff --git a/lib/PLEGMA_Field.cu b/lib/PLEGMA_Field.cu index e5070bc5..b14e025b 100644 --- a/lib/PLEGMA_Field.cu +++ b/lib/PLEGMA_Field.cu @@ -1,19 +1,20 @@ #include #include -#include -#include +#include +#include #include #include #include #include #include #include -#include +#include #include #include #include #include #include +#include #include #include using namespace plegma; @@ -347,7 +348,7 @@ void PLEGMA_Field::zero_where(ALLOCATION_FLAG alloc_flag){ PLEGMA_error("Not supported %d\n",alloc_flag); } } - +#ifdef __NVCC__ template cudaTextureObject_t PLEGMA_Field::createTexObject() const{ #ifdef PLEGMA_TEXTURE @@ -395,6 +396,57 @@ void PLEGMA_Field::destroyTexObject(cudaTextureObject_t tex) const{ cudaDestroyTextureObject(tex); #endif } +#endif + +#ifdef __HIP__ +template +hipTextureObject_t PLEGMA_Field::createTexObject() const{ +#ifdef PLEGMA_TEXTURE + hipTextureObject_t tex; + hipChannelFormatDesc desc; + memset(&desc, 0, sizeof(hipChannelFormatDesc)); + int precision = PLEGMA_Field::Precision(); + if(precision == 4) desc.f = hipChannelFormatKindFloat; + else desc.f = hipChannelFormatKindSigned; + + if(precision == 4){ + desc.x = 8*precision; + desc.y = 8*precision; + desc.z = 0; + desc.w = 0; + } + else if(precision == 8){ + desc.x = 8*precision/2; + desc.y = 8*precision/2; + desc.z = 8*precision/2; + desc.w = 8*precision/2; + } + + hipResourceDesc resDesc; + memset(&resDesc, 0, sizeof(resDesc)); + resDesc.resType = hipResourceTypeLinear; + resDesc.res.linear.devPtr = d_elem; + resDesc.res.linear.desc = desc; + resDesc.res.linear.sizeInBytes = Bytes_total_plus_ghost(); + + hipTextureDesc texDesc; + memset(&texDesc, 0, sizeof(texDesc)); + texDesc.readMode = hipReadModeElementType; + + hipCreateTextureObject(&tex, &resDesc, &texDesc, NULL); + return tex; +#else + return 0; +#endif +} + +template +void PLEGMA_Field::destroyTexObject(hipTextureObject_t tex) const{ +#ifdef PLEGMA_TEXTURE + hipDestroyTextureObject(tex); +#endif +} +#endif template void PLEGMA_Field::printInfo(){ @@ -1066,8 +1118,8 @@ template void PLEGMA_Field::SU3Trace(PLEGMA_Su3field &su3field){ SU3Trace_k(*this,su3field); } -template class PLEGMA_Field; -template class PLEGMA_Field; +template class plegma::PLEGMA_Field; +template class plegma::PLEGMA_Field; // Forcing initialization of the following cases template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); template void PLEGMA_Field::copy(PLEGMA_Field &f, ALLOCATION_FLAG where); @@ -1140,5 +1192,5 @@ std::complex PLEGMA_Field3D::dot(PLEGMA_Field3D &fieldIn){ return cuBLAS::dot(this->total_length*this->field_length, this->d_elem, fieldIn.D_elem(), HGC_fullComm); } - template class PLEGMA_Field3D; -template class PLEGMA_Field3D; +template class plegma::PLEGMA_Field3D; +template class plegma::PLEGMA_Field3D; diff --git a/lib/PLEGMA_Fmunu.cu b/lib/PLEGMA_Fmunu.cu index d405ec05..1bea5dc4 100644 --- a/lib/PLEGMA_Fmunu.cu +++ b/lib/PLEGMA_Fmunu.cu @@ -1,6 +1,6 @@ #include #include -#include +#include using namespace plegma; //---------------------------// // class PLEGMA_Fmunu // @@ -20,5 +20,5 @@ void PLEGMA_Fmunu::compute_leaves(PLEGMA_Gauge &gauge){ this->cscale(coeff); } -template class PLEGMA_Fmunu; -template class PLEGMA_Fmunu; +template class plegma::PLEGMA_Fmunu; +template class plegma::PLEGMA_Fmunu; diff --git a/lib/PLEGMA_Gauge.cu b/lib/PLEGMA_Gauge.cu index 46fb2d4e..236eafa9 100644 --- a/lib/PLEGMA_Gauge.cu +++ b/lib/PLEGMA_Gauge.cu @@ -1,13 +1,13 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include using namespace plegma; @@ -282,8 +282,8 @@ void PLEGMA_Gauge::U3xU1(PLEGMA_Gauge &u3, PLEGMA_U1Gauge & U3xU1_k(*this,u3,u1); } -template class PLEGMA_Gauge; -template class PLEGMA_Gauge; +template class plegma::PLEGMA_Gauge; +template class plegma::PLEGMA_Gauge; -template class PLEGMA_Gauge3D; -template class PLEGMA_Gauge3D; +template class plegma::PLEGMA_Gauge3D; +template class plegma::PLEGMA_Gauge3D; diff --git a/lib/PLEGMA_Propagator.cu b/lib/PLEGMA_Propagator.cu index f7cdfa69..dc9f4412 100644 --- a/lib/PLEGMA_Propagator.cu +++ b/lib/PLEGMA_Propagator.cu @@ -1,6 +1,6 @@ #include #include -#include +#include using namespace plegma; //-------------------------------// @@ -344,7 +344,7 @@ void PLEGMA_Propagator3D::absorb(PLEGMA_Vector3D &vec, int nu, int checkQudaError(); } -template class PLEGMA_Propagator; -template class PLEGMA_Propagator3D; -template class PLEGMA_Propagator; -template class PLEGMA_Propagator3D; +template class plegma::PLEGMA_Propagator; +template class plegma::PLEGMA_Propagator3D; +template class plegma::PLEGMA_Propagator; +template class plegma::PLEGMA_Propagator3D; diff --git a/lib/PLEGMA_QLoops.cu b/lib/PLEGMA_QLoops.cu index 6c61266f..ba5fe735 100644 --- a/lib/PLEGMA_QLoops.cu +++ b/lib/PLEGMA_QLoops.cu @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include using namespace plegma; @@ -319,9 +319,9 @@ void PLEGMA_QLoops::write_ASCII(std::string filename_local, std::string f template void PLEGMA_QLoops::load(Float* h_ptr){ - cudaMemcpy(this->D_elem(), h_ptr, this->Bytes_total(), cudaMemcpyHostToDevice ); + qudaMemcpy(this->D_elem(), h_ptr, this->Bytes_total(), qudaMemcpyHostToDevice ); } -template class PLEGMA_QLoops; -template class PLEGMA_QLoops; +template class plegma::PLEGMA_QLoops; +template class plegma::PLEGMA_QLoops; diff --git a/lib/PLEGMA_Random.cu b/lib/PLEGMA_Random.cu index 408c3517..8b4fd8fc 100644 --- a/lib/PLEGMA_Random.cu +++ b/lib/PLEGMA_Random.cu @@ -1,5 +1,6 @@ #include -#include +#include +//#include using namespace plegma; //--------------------------// @@ -53,6 +54,7 @@ PLEGMA_RNG::PLEGMA_RNG(int seedin, int rng_sizes) { void PLEGMA_RNG::Init() { //printf("Number of rng_size[2]: %d\n", rng_size); AllocateRNG(); + //random_init(seed, (unsigned long long)sequence, offset, state); launch_random_init(state, seed, rank_offset, rng_size); } @@ -63,9 +65,9 @@ void PLEGMA_RNG::AllocateRNG() { //printf("Number of rng_size[1.5]: %d\n", rng_size); if (rng_size>0 && state == NULL) { - cudaMalloc((void**)&state, rng_size * sizeof(cuRNGState)); - cudaMemset( state , 0 , rng_size * sizeof(cuRNGState) ); - PLEGMA_printf("Allocated array of random numbers with rng_size: %.2f MB\n",((float)rng_size * (float)sizeof(cuRNGState))/(1024*1024)); + state=(RNGState*)device_malloc(rng_size * sizeof(RNGState)); + qudaMemset( state , 0 , rng_size * sizeof(RNGState) ); + PLEGMA_printf("Allocated array of random numbers with rng_size: %.2f MB\n",((float)rng_size * (float)sizeof(RNGState))/(1024*1024)); } else { PLEGMA_error("Array of random numbers not allocated, array size: %d !\nExiting...\n",rng_size); } @@ -73,8 +75,8 @@ void PLEGMA_RNG::AllocateRNG() { /*! @brief Destructor !*/ PLEGMA_RNG::~PLEGMA_RNG(){ - cudaFree(state); - PLEGMA_printf("Free array of random numbers with rng_size: %.2f MB\n", ((float)rng_size * (float)sizeof(cuRNGState))/(1024*1024)); + device_free(state); + PLEGMA_printf("Free array of random numbers with rng_size: %.2f MB\n", ((float)rng_size * (float)sizeof(RNGState))/(1024*1024)); rng_size = 0; state = NULL; checkQudaError(); @@ -83,15 +85,15 @@ PLEGMA_RNG::~PLEGMA_RNG(){ /*! @brief Generating random numbers from random distribution */ /*! @brief Restore CURAND array states initialization */ void PLEGMA_RNG::restore() { - cudaMemcpy(state, backup_state, rng_size * sizeof(cuRNGState), cudaMemcpyHostToDevice); + qudaMemcpy(state, backup_state, rng_size * sizeof(RNGState), qudaMemcpyHostToDevice); checkQudaError(); - hostFree(backup_state, rng_size * sizeof(cuRNGState)); + hostFree(backup_state, rng_size * sizeof(RNGState)); } /*! @brief Backup CURAND array states initialization */ void PLEGMA_RNG::backup() { - hostMalloc(backup_state, rng_size * sizeof(cuRNGState)); - cudaMemcpy(backup_state, state, rng_size * sizeof(cuRNGState), cudaMemcpyDeviceToHost); + hostMalloc(backup_state, rng_size * sizeof(RNGState)); + qudaMemcpy(backup_state, state, rng_size * sizeof(RNGState), qudaMemcpyDeviceToHost); checkQudaError(); } diff --git a/lib/PLEGMA_ScattCorrelator.cu b/lib/PLEGMA_ScattCorrelator.cu index b5169d6a..30f725c1 100644 --- a/lib/PLEGMA_ScattCorrelator.cu +++ b/lib/PLEGMA_ScattCorrelator.cu @@ -1,16 +1,15 @@ #include #include #include -#include -#include +#include +#include #include #include -#include -#include -#include +#include +//#include using namespace plegma; using namespace quda; - +#include Communicator &get_current_communicator(); @@ -416,7 +415,7 @@ Float *PLEGMA_ScattCorrelator::get_time_slice(int global_time_index){ int coords[4]; for(int i = 0 ; i < (N_DIMS-1); i++) coords[i] = 0; coords[N_DIMS-1]=global_time_index / HGC_localL[N_DIMS-1]; - int rankHas = quda::comm_rank_from_coords(coords); + int rankHas = comm_rank_from_coords(coords); printf("rankHas %d\n",rankHas); MPI_Barrier(HGC_fullComm); int mpiErr = MPI_Bcast(ptr, size_timeslice, MPI_Type(), rankHas, HGC_fullComm); @@ -443,7 +442,7 @@ std::shared_ptr PLEGMA_ScattCorrelator::average_all_time_slices(){ size_timeslice *= 2; } Float *localsum=(Float *)malloc(sizeof(Float)*size_timeslice); - std::shared_ptr ptr((Float *)malloc(sizeof(Float)*size_timeslice), free); + std::shared_ptr ptr(new Float[size_timeslice]); for(int j=0; jlocalT(); @@ -3730,7 +3729,7 @@ void PLEGMA_ScattCorrelator::absorbTimeslice(PLEGMA_ScattCorrelatorH_elem(), in_dofs_src*out_dofs_src , MPI_Type(), rankHas, HGC_fullComm); if(mpiErr != MPI_SUCCESS) PLEGMA_error("MPI_Bcast failed with error %d\n", mpiErr); @@ -3873,7 +3872,7 @@ void PLEGMA_ScattCorrelator::apply_sign(std::string name_of_diagram ){ } } -template class PLEGMA_ScattCorrelator; -template class PLEGMA_ScattCorrelator; +template class plegma::PLEGMA_ScattCorrelator; +template class plegma::PLEGMA_ScattCorrelator; diff --git a/lib/PLEGMA_Su3field.cu b/lib/PLEGMA_Su3field.cu index 7111574f..4c1bc1e2 100644 --- a/lib/PLEGMA_Su3field.cu +++ b/lib/PLEGMA_Su3field.cu @@ -1,8 +1,8 @@ #include #include -#include -#include -#include +#include +#include +#include #include using namespace plegma; @@ -166,5 +166,5 @@ void PLEGMA_Su3field::wilsonLineUpdate(PLEGMA_Su3field &inOut, PLE } -template class PLEGMA_Su3field; -template class PLEGMA_Su3field; +template class plegma::PLEGMA_Su3field; +template class plegma::PLEGMA_Su3field; diff --git a/lib/PLEGMA_U1Gauge.cu b/lib/PLEGMA_U1Gauge.cu index 90a0ee52..687b3c7a 100644 --- a/lib/PLEGMA_U1Gauge.cu +++ b/lib/PLEGMA_U1Gauge.cu @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include using namespace plegma; @@ -52,5 +52,5 @@ void PLEGMA_U1Gauge::modifyBoundaries(int mu, int nu, Float exparg){ this->load(); } -template class PLEGMA_U1Gauge; -template class PLEGMA_U1Gauge; +template class plegma::PLEGMA_U1Gauge; +template class plegma::PLEGMA_U1Gauge; diff --git a/lib/PLEGMA_Vector.cu b/lib/PLEGMA_Vector.cu index aca2d39e..8d182d4e 100644 --- a/lib/PLEGMA_Vector.cu +++ b/lib/PLEGMA_Vector.cu @@ -2,15 +2,16 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #ifdef PLEGMA_SCATTERING_CONTRACTIONS #include #include #endif #include +#include #include #include using namespace plegma; @@ -416,7 +417,7 @@ void PLEGMA_Vector::pointSource(const site& sourceposition, int spin, int template std::shared_ptr PLEGMA_Vector::getPointSource( const site& sourceposition, ALLOCATION_FLAG where){ if (where == HOST){ - std::shared_ptr ptr((Float *)malloc(sizeof(Float)*N_SPINS*N_COLS*2), free); + std::shared_ptr ptr(new Float[N_SPINS*N_COLS*2]); for(int i = 0; i < N_DIMS; i++) if(sourceposition[i] >= HGC_totalL[i]) PLEGMA_error("Source position component in dir=%d, is %d >= %d the lattice extent", i, sourceposition[i],HGC_totalL[i]); @@ -486,8 +487,8 @@ void PLEGMA_Vector::mulGV(PLEGMA_Vector &vecIn, PLEGMA_Su3field; -template class PLEGMA_Vector; +template class plegma::PLEGMA_Vector; +template class plegma::PLEGMA_Vector; namespace plegma{ @@ -671,6 +672,6 @@ namespace plegma{ } - template class PLEGMA_Vector3D; - template class PLEGMA_Vector3D; + template class plegma::PLEGMA_Vector3D; + template class plegma::PLEGMA_Vector3D; } diff --git a/lib/kernels/PLEGMA_FT.cuh b/lib/kernels/PLEGMA_FT.cuh index 80f01a90..40bae8e1 100644 --- a/lib/kernels/PLEGMA_FT.cuh +++ b/lib/kernels/PLEGMA_FT.cuh @@ -1,6 +1,7 @@ +#pragma once #include #include -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template @@ -54,8 +55,8 @@ static void FT_dot(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vect int V3 = HGC_localVolume/HGC_localL[3]; int V = ft.Dims() == 3 ? V3 : HGC_localVolume; Float2 *x; - cudaMalloc((void**)&x, V*2*sizeof(Float)); - cudaMemset(x,0,V*2*sizeof(Float)); + x=(Float*)device_malloc(V*2*sizeof(Float)); + qudaMemset(x,0,V*2*sizeof(Float)); checkQudaError(); for(int imom = 0; imom < Nmom; imom++){ createMomField(x,mom[imom],ft.Dims(),-sign); // change sign to compensate dagger @@ -68,7 +69,7 @@ static void FT_dot(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vect ft.H_elem()[it*f.Field_length()*Nmom*2 + idf*Nmom*2 + imom*2 + 1] += res.imag(); } } - cudaFree(x); + device_free(x); } template @@ -79,9 +80,9 @@ static void FT_gemv(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vec int V3 = HGC_localVolume/HGC_localL[3]; int V = ft.Dims() == 3 ? V3 : HGC_localVolume; Float2 *x,*d_res; - cudaMalloc((void**)&x, V*2*sizeof(Float)); - cudaMemset(x,0,V*2*sizeof(Float)); - cudaMalloc((void**)&d_res, f.Field_length() * ft.DimT() * 2*sizeof(Float)); + x=(Float2*)device_malloc(V*2*sizeof(Float)); + qudaMemset(x,0,V*2*sizeof(Float)); + d_res=(Float2*)device_malloc(f.Field_length() * ft.DimT() * 2*sizeof(Float2)); checkQudaError(); Float2 h_res[f.Field_length()*ft.DimT()]; Float2 *h_ft = (Float2 *) ft.H_elem(); @@ -95,8 +96,8 @@ static void FT_gemv(PLEGMA_FT &ft, const PLEGMA_Field &f, std::vec for(int it = 0 ; it < ft.DimT(); it++) h_ft[it*f.Field_length()*Nmom + idf*Nmom + imom] += h_res[idf*ft.DimT()+it]; } - cudaFree(x); - cudaFree(d_res); + device_free(x); + device_free(d_res); } @@ -130,14 +131,14 @@ static void fourier_transform_3D_k(PLEGMA_FT &ft, const PLEGMA_Field, d_partial_block, toField2(field), texMomList, it, sign); size_t alloc_size=ft.Nmoms()*site_size*ps.tp.grid.x*2*sizeof(Float); - cudaMalloc((void**)&d_partial_block, alloc_size); + d_partial_block=(Float *)device_malloc( alloc_size); checkQudaError(); run(ps, "fourier_transform_3D_kernel", fourier_transform_3D_kernel, d_partial_block, toField2(field), texMomList, it, sign); Float *h_partial_block = NULL; hostMalloc(h_partial_block,alloc_size); - cudaMemcpy(h_partial_block,d_partial_block,alloc_size,cudaMemcpyDeviceToHost); + qudaMemcpy(h_partial_block,d_partial_block,alloc_size,qudaMemcpyDeviceToHost); checkQudaError(); - cudaFree(d_partial_block); + device_free(d_partial_block); int gridDimX = ps.tp.grid.x; Float *reduction; hostMalloc(reduction,ft.Nmoms()*site_size*2*sizeof(Float)); diff --git a/lib/kernels/PLEGMA_QWF.cuh b/lib/kernels/PLEGMA_QWF.cuh index 66e58843..343243e1 100644 --- a/lib/kernels/PLEGMA_QWF.cuh +++ b/lib/kernels/PLEGMA_QWF.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template @@ -12,7 +12,7 @@ __global__ void contract_TMDWF_mesons_trick_zfac_device(propTextexProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[16]; + Float2 accum[16]; #pragma unroll for(int i = 0 ; i < 16 ; i++){ accum[i] = 0.; @@ -71,7 +71,7 @@ __global__ void contract_TMDWF_mesons_zfac_device(propTextexProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[16]; + Float2 accum[16]; #pragma unroll for(int i = 0 ; i < 16 ; i++){ accum[i] = 0.; @@ -149,16 +149,16 @@ void contract_TMDWF_mesons_trick_zfac_host( ProfileStruct &ps,PLEGMA_Propagator< Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); auto stapleTex = toTexture(staple); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) { + //cudaError_t error=cudaPeekAtLastError(); + if(h_partial_block==NULL) { hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); return; } @@ -173,10 +173,10 @@ void contract_TMDWF_mesons_trick_zfac_host( ProfileStruct &ps,PLEGMA_Propagator< contract_TMDWF_mesons_trick_zfac_device <<>> (*propTex1, *stapleTex, mu, nu, c1, c2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + // error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); + // error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -197,7 +197,7 @@ void contract_TMDWF_mesons_trick_zfac_host( ProfileStruct &ps,PLEGMA_Propagator< } hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template @@ -223,17 +223,17 @@ void contract_TMDWF_mesons_zfac_host( ProfileStruct &ps,PLEGMA_Propagator *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); auto propTex2 = toTexture(prop2); auto stapleTex = toTexture(staple); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) { + //cudaError_t error=cudaPeekAtLastError(); + if(h_partial_block==NULL) { hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); return; } @@ -248,10 +248,10 @@ void contract_TMDWF_mesons_zfac_host( ProfileStruct &ps,PLEGMA_Propagator>> (*propTex1, *propTex2, *stapleTex, mu, nu, c1, c2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -272,7 +272,7 @@ void contract_TMDWF_mesons_zfac_host( ProfileStruct &ps,PLEGMA_Propagator diff --git a/lib/kernels/PLEGMA_Random.cuh b/lib/kernels/PLEGMA_Random.cuh index 9f175179..a7080ad5 100644 --- a/lib/kernels/PLEGMA_Random.cuh +++ b/lib/kernels/PLEGMA_Random.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" #include @@ -10,7 +10,7 @@ using namespace plegma; @param length_field length of the CURAND RNG state array @param offset offset of the RNG sequence */ -__global__ void random_init_kernel(cuRNGState *state, int seed, int offset){ +__global__ void random_init_kernel(RNGState *state, int seed, int offset){ int sid = blockIdx.x*blockDim.x + threadIdx.x; //printf("Field length %d", length_field); @@ -18,8 +18,8 @@ __global__ void random_init_kernel(cuRNGState *state, int seed, int offset){ //Determine the global id of the field. int seq_number = LEXIC_1DL_1DG(sid); //printf("Number of threads %d and seq number %d\n", sid, seq_number); - - curand_init(seed, seq_number, offset, &state[sid]); + + random_init(seed, seq_number, offset, state[sid]); } /** @@ -29,11 +29,11 @@ __global__ void random_init_kernel(cuRNGState *state, int seed, int offset){ @param field_deg_free Degrees of Freedom of the field @param offset offset of the RNG sequence */ -void launch_random_init( cuRNGState *state, int seed, int offset, int rng_size){ +void launch_random_init( RNGState *state, int seed, int offset, int rng_size){ dim3 blockDim( THREADS_PER_BLOCK, 1, 1); //PLEGMA_printf("Number of volume[3]: %d\n", HGC_localVolume * field_deg_free); dim3 gridDim( (rng_size + blockDim.x -1)/blockDim.x , 1 , 1); random_init_kernel<<>>(state, seed, offset ); - cudaDeviceSynchronize(); + //cudaDeviceSynchronize(); } diff --git a/lib/kernels/PLEGMA_SU3_projection.cuh b/lib/kernels/PLEGMA_SU3_projection.cuh index 30020c3f..9fb5263b 100644 --- a/lib/kernels/PLEGMA_SU3_projection.cuh +++ b/lib/kernels/PLEGMA_SU3_projection.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; // input is M and output is U diff --git a/lib/kernels/PLEGMA_TMDWF.cuh b/lib/kernels/PLEGMA_TMDWF.cuh index 555aebcc..bc4e253a 100644 --- a/lib/kernels/PLEGMA_TMDWF.cuh +++ b/lib/kernels/PLEGMA_TMDWF.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" #include #include using namespace plegma; @@ -24,7 +24,7 @@ __global__ void contract_TMDWF_mesons_device( propTex texProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[2*N_TMDWF_MESONS]; + Float2 accum[2*N_TMDWF_MESONS]; for(int i = 0 ; i < 2*N_TMDWF_MESONS ; i++){ accum[i] = 0.; } @@ -99,11 +99,11 @@ void contract_TMDWF_mesons_host( ProfileStruct &ps, //cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); d_partial_block=(Float2 *)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { - cudaFree(d_partial_block); - return; - } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { + // cudaFree(d_partial_block); + // return; + //} hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -115,10 +115,10 @@ void contract_TMDWF_mesons_host( ProfileStruct &ps, contract_TMDWF_mesons_device <<>> (*propTex1, *propTex2, *stapleTex ,d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_WFlow.cuh b/lib/kernels/PLEGMA_WFlow.cuh index 8f472531..8dfdf333 100644 --- a/lib/kernels/PLEGMA_WFlow.cuh +++ b/lib/kernels/PLEGMA_WFlow.cuh @@ -1,6 +1,6 @@ #pragma once -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" #include using namespace plegma; @@ -17,22 +17,22 @@ __device__ void calculatestaples(Float2 U[N_COLS][N_COLS], gauge2(u1,dir,sid,nu); // u1 = U(i+\nu)^(\mu) + gaugep.template get(u1,dir,sid,nu); // u1 = U(i+\nu)^(\mu) gaugep.get(u2, nu, sid); // u2 = U(i)^(\nu) mul_Gdag_Gdag(aux,u1,u2); // aux = u1+*u2+ - gaugep.get(u2,nu,sid,dir); // u2 = U(i+\mu)^(\nu) + gaugep.template get(u2,nu,sid,dir); // u2 = U(i+\mu)^(\nu) mul_G_G(u1,u2,aux); // u1 = U(i+\mu)^(\nu)*U(i+\nu)^(+\mu)*U(i)^(+\nu) = staple_fwd G_plus_aG(U, u1, 1.);//U += u1 //bwd //nu+=4; - gaugep.get(u1,nu,sid,dir,nu); // u1 = U(i+\mu-\nu)^(\nu) - gaugep.get(u2, dir, sid, nu); // u2 = U(i-\nu)^(\mu) + gaugep.template get(u1,nu,sid,dir,nu); // u1 = U(i+\mu-\nu)^(\nu) + gaugep.template get(u2, dir, sid, nu); // u2 = U(i-\nu)^(\mu) mul_Gdag_Gdag(aux,u1,u2); // aux = u1+*u2+ - gaugep.get(u2, nu, sid, nu); // u2 = U(i-\nu)^(\nu) + gaugep.template get(u2, nu, sid, nu); // u2 = U(i-\nu)^(\nu) mul_G_G(u1,aux,u2); // u1 = U(i+\mu-\nu)^(+\nu)*U(i-\nu)^(+\mu)*U(i-\nu)^(\nu) = staple_bwd G_plus_aG( U, u1, 1.); //U += staple @@ -109,7 +109,7 @@ __inline__ void GFlow_substep( gauge2 W, gauge2 Z, FloatE e_work //FloatG* debug_array; ZUpdate<<>>( W, Z, e_work, e_save ); - cudaDeviceSynchronize(); +// cudaDeviceSynchronize(); WUpdate<<>>( W, Z ); checkQudaError(); @@ -197,8 +197,8 @@ static FloatG calcPlaqStaplesDef(gauge2 gaugep){ calcPlaqStaplesDef_kernel<<>>( gaugep, d_partial_plaq ); - cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDim.x * sizeof(FloatG) , cudaMemcpyDeviceToHost); - cudaFree(d_partial_plaq); + qudaMemcpy(h_partial_plaq, d_partial_plaq , gridDim.x * sizeof(FloatG) , qudaMemcpyDeviceToHost); + device_free(d_partial_plaq); checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++) diff --git a/lib/kernels/PLEGMA_baryons.cuh b/lib/kernels/PLEGMA_baryons.cuh index 6426b788..445dcc18 100644 --- a/lib/kernels/PLEGMA_baryons.cuh +++ b/lib/kernels/PLEGMA_baryons.cuh @@ -1,6 +1,6 @@ #pragma once -#include +#include "PLEGMA_kernel_utils.cuh" #include enum BARYONS_TYPE{NtoN, #ifdef PLEGMA_LIGHT_BARYONS @@ -192,11 +192,11 @@ static void contract_baryons_host( ProfileStruct &ps, d_partial_block=(Float2*)device_malloc(sizeof(Float2) ); // cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); + /*cudaError_t error=cudaPeekAtLastError(); if(error != cudaSuccess) { cudaFree(d_partial_block); return; - } + }*/ hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -209,10 +209,10 @@ static void contract_baryons_host( ProfileStruct &ps, <<>> (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, (BARYONS_TYPE) ip, runFT, *mom_list); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2) , qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -234,7 +234,7 @@ static void contract_baryons_host( ProfileStruct &ps, } } hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template @@ -263,11 +263,11 @@ static void contract_baryons_wall_host( ProfileStruct &ps, //cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); d_partial_block=(Float2*)device_malloc(sizeof(Float2) ); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); + /*cudaError_t error=cudaPeekAtLastError(); if(error != cudaSuccess) { cudaFree(d_partial_block); return; - } + }*/ hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -282,10 +282,10 @@ static void contract_baryons_wall_host( ProfileStruct &ps, <<>> (*propTex1, *propTex2, *propTex3, *propTex4, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, (BARYONS_TYPE) ip, runFT, *mom_list); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2) , qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -307,7 +307,7 @@ static void contract_baryons_wall_host( ProfileStruct &ps, } } hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template diff --git a/lib/kernels/PLEGMA_baryons_NtoN.cu b/lib/kernels/PLEGMA_baryons_NtoN.cu index b2873a5e..1e1ceb9d 100644 --- a/lib/kernels/PLEGMA_baryons_NtoN.cu +++ b/lib/kernels/PLEGMA_baryons_NtoN.cu @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" static const __device__ short int NtoN_indices[16][4] = {0,1,0,1,0,1,1,0,0,1,2,3,0,1,3,2,1,0,0,1,1,0,1,0,1,0,2,3,1,0,3,2,2,3,0,1,2,3,1,0,2,3,2,3,2,3,3,2,3,2,0,1,3,2,1,0,3,2,2,3,3,2,3,2}; static const __device__ float NtoN_values[16] = {-1,1,-1,1,1,-1,1,-1,-1,1,-1,1,1,-1,1,-1}; diff --git a/lib/kernels/PLEGMA_baryons_NtoR.cu b/lib/kernels/PLEGMA_baryons_NtoR.cu index 23463a42..5e17a025 100644 --- a/lib/kernels/PLEGMA_baryons_NtoR.cu +++ b/lib/kernels/PLEGMA_baryons_NtoR.cu @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" static const __device__ short int NTR_indices[64][6] = {0,1,0,3,0,2,0,1,0,3,1,3,0,1,0,3,2,0,0,1,0,3,3,1,0,1,1,2,0,2,0,1,1,2,1,3,0,1,1,2,2,0,0,1,1,2,3,1,0,1,2,1,0,2,0,1,2,1,1,3,0,1,2,1,2,0,0,1,2,1,3,1,0,1,3,0,0,2,0,1,3,0,1,3,0,1,3,0,2,0,0,1,3,0,3,1,1,0,0,3,0,2,1,0,0,3,1,3,1,0,0,3,2,0,1,0,0,3,3,1,1,0,1,2,0,2,1,0,1,2,1,3,1,0,1,2,2,0,1,0,1,2,3,1,1,0,2,1,0,2,1,0,2,1,1,3,1,0,2,1,2,0,1,0,2,1,3,1,1,0,3,0,0,2,1,0,3,0,1,3,1,0,3,0,2,0,1,0,3,0,3,1,2,3,0,3,0,2,2,3,0,3,1,3,2,3,0,3,2,0,2,3,0,3,3,1,2,3,1,2,0,2,2,3,1,2,1,3,2,3,1,2,2,0,2,3,1,2,3,1,2,3,2,1,0,2,2,3,2,1,1,3,2,3,2,1,2,0,2,3,2,1,3,1,2,3,3,0,0,2,2,3,3,0,1,3,2,3,3,0,2,0,2,3,3,0,3,1,3,2,0,3,0,2,3,2,0,3,1,3,3,2,0,3,2,0,3,2,0,3,3,1,3,2,1,2,0,2,3,2,1,2,1,3,3,2,1,2,2,0,3,2,1,2,3,1,3,2,2,1,0,2,3,2,2,1,1,3,3,2,2,1,2,0,3,2,2,1,3,1,3,2,3,0,0,2,3,2,3,0,1,3,3,2,3,0,2,0,3,2,3,0,3,1}; static const __device__ float NTR_values[64] = {1,1,1,1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,1,1,1,1}; diff --git a/lib/kernels/PLEGMA_baryons_RtoN.cu b/lib/kernels/PLEGMA_baryons_RtoN.cu index 66e29a01..8cc534f2 100644 --- a/lib/kernels/PLEGMA_baryons_RtoN.cu +++ b/lib/kernels/PLEGMA_baryons_RtoN.cu @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" static const __device__ short int RtoN_indices[64][6] = {0,3,0,1,0,2,0,3,0,1,1,3,0,3,0,1,2,0,0,3,0,1,3,1,0,3,1,0,0,2,0,3,1,0,1,3,0,3,1,0,2,0,0,3,1,0,3,1,0,3,2,3,0,2,0,3,2,3,1,3,0,3,2,3,2,0,0,3,2,3,3,1,0,3,3,2,0,2,0,3,3,2,1,3,0,3,3,2,2,0,0,3,3,2,3,1,1,2,0,1,0,2,1,2,0,1,1,3,1,2,0,1,2,0,1,2,0,1,3,1,1,2,1,0,0,2,1,2,1,0,1,3,1,2,1,0,2,0,1,2,1,0,3,1,1,2,2,3,0,2,1,2,2,3,1,3,1,2,2,3,2,0,1,2,2,3,3,1,1,2,3,2,0,2,1,2,3,2,1,3,1,2,3,2,2,0,1,2,3,2,3,1,2,1,0,1,0,2,2,1,0,1,1,3,2,1,0,1,2,0,2,1,0,1,3,1,2,1,1,0,0,2,2,1,1,0,1,3,2,1,1,0,2,0,2,1,1,0,3,1,2,1,2,3,0,2,2,1,2,3,1,3,2,1,2,3,2,0,2,1,2,3,3,1,2,1,3,2,0,2,2,1,3,2,1,3,2,1,3,2,2,0,2,1,3,2,3,1,3,0,0,1,0,2,3,0,0,1,1,3,3,0,0,1,2,0,3,0,0,1,3,1,3,0,1,0,0,2,3,0,1,0,1,3,3,0,1,0,2,0,3,0,1,0,3,1,3,0,2,3,0,2,3,0,2,3,1,3,3,0,2,3,2,0,3,0,2,3,3,1,3,0,3,2,0,2,3,0,3,2,1,3,3,0,3,2,2,0,3,0,3,2,3,1}; static const __device__ float RtoN_values[64] = {-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1}; diff --git a/lib/kernels/PLEGMA_baryons_RtoR.cu b/lib/kernels/PLEGMA_baryons_RtoR.cu index 440b95f6..0772d004 100644 --- a/lib/kernels/PLEGMA_baryons_RtoR.cu +++ b/lib/kernels/PLEGMA_baryons_RtoR.cu @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" static const __device__ short int RTR_indices[256][8] = {0,3,0,3,0,2,0,2,0,3,0,3,0,2,1,3,0,3,0,3,0,2,2,0,0,3,0,3,0,2,3,1,0,3,0,3,1,3,0,2,0,3,0,3,1,3,1,3,0,3,0,3,1,3,2,0,0,3,0,3,1,3,3,1,0,3,0,3,2,0,0,2,0,3,0,3,2,0,1,3,0,3,0,3,2,0,2,0,0,3,0,3,2,0,3,1,0,3,0,3,3,1,0,2,0,3,0,3,3,1,1,3,0,3,0,3,3,1,2,0,0,3,0,3,3,1,3,1,0,3,1,2,0,2,0,2,0,3,1,2,0,2,1,3,0,3,1,2,0,2,2,0,0,3,1,2,0,2,3,1,0,3,1,2,1,3,0,2,0,3,1,2,1,3,1,3,0,3,1,2,1,3,2,0,0,3,1,2,1,3,3,1,0,3,1,2,2,0,0,2,0,3,1,2,2,0,1,3,0,3,1,2,2,0,2,0,0,3,1,2,2,0,3,1,0,3,1,2,3,1,0,2,0,3,1,2,3,1,1,3,0,3,1,2,3,1,2,0,0,3,1,2,3,1,3,1,0,3,2,1,0,2,0,2,0,3,2,1,0,2,1,3,0,3,2,1,0,2,2,0,0,3,2,1,0,2,3,1,0,3,2,1,1,3,0,2,0,3,2,1,1,3,1,3,0,3,2,1,1,3,2,0,0,3,2,1,1,3,3,1,0,3,2,1,2,0,0,2,0,3,2,1,2,0,1,3,0,3,2,1,2,0,2,0,0,3,2,1,2,0,3,1,0,3,2,1,3,1,0,2,0,3,2,1,3,1,1,3,0,3,2,1,3,1,2,0,0,3,2,1,3,1,3,1,0,3,3,0,0,2,0,2,0,3,3,0,0,2,1,3,0,3,3,0,0,2,2,0,0,3,3,0,0,2,3,1,0,3,3,0,1,3,0,2,0,3,3,0,1,3,1,3,0,3,3,0,1,3,2,0,0,3,3,0,1,3,3,1,0,3,3,0,2,0,0,2,0,3,3,0,2,0,1,3,0,3,3,0,2,0,2,0,0,3,3,0,2,0,3,1,0,3,3,0,3,1,0,2,0,3,3,0,3,1,1,3,0,3,3,0,3,1,2,0,0,3,3,0,3,1,3,1,1,2,0,3,0,2,0,2,1,2,0,3,0,2,1,3,1,2,0,3,0,2,2,0,1,2,0,3,0,2,3,1,1,2,0,3,1,3,0,2,1,2,0,3,1,3,1,3,1,2,0,3,1,3,2,0,1,2,0,3,1,3,3,1,1,2,0,3,2,0,0,2,1,2,0,3,2,0,1,3,1,2,0,3,2,0,2,0,1,2,0,3,2,0,3,1,1,2,0,3,3,1,0,2,1,2,0,3,3,1,1,3,1,2,0,3,3,1,2,0,1,2,0,3,3,1,3,1,1,2,1,2,0,2,0,2,1,2,1,2,0,2,1,3,1,2,1,2,0,2,2,0,1,2,1,2,0,2,3,1,1,2,1,2,1,3,0,2,1,2,1,2,1,3,1,3,1,2,1,2,1,3,2,0,1,2,1,2,1,3,3,1,1,2,1,2,2,0,0,2,1,2,1,2,2,0,1,3,1,2,1,2,2,0,2,0,1,2,1,2,2,0,3,1,1,2,1,2,3,1,0,2,1,2,1,2,3,1,1,3,1,2,1,2,3,1,2,0,1,2,1,2,3,1,3,1,1,2,2,1,0,2,0,2,1,2,2,1,0,2,1,3,1,2,2,1,0,2,2,0,1,2,2,1,0,2,3,1,1,2,2,1,1,3,0,2,1,2,2,1,1,3,1,3,1,2,2,1,1,3,2,0,1,2,2,1,1,3,3,1,1,2,2,1,2,0,0,2,1,2,2,1,2,0,1,3,1,2,2,1,2,0,2,0,1,2,2,1,2,0,3,1,1,2,2,1,3,1,0,2,1,2,2,1,3,1,1,3,1,2,2,1,3,1,2,0,1,2,2,1,3,1,3,1,1,2,3,0,0,2,0,2,1,2,3,0,0,2,1,3,1,2,3,0,0,2,2,0,1,2,3,0,0,2,3,1,1,2,3,0,1,3,0,2,1,2,3,0,1,3,1,3,1,2,3,0,1,3,2,0,1,2,3,0,1,3,3,1,1,2,3,0,2,0,0,2,1,2,3,0,2,0,1,3,1,2,3,0,2,0,2,0,1,2,3,0,2,0,3,1,1,2,3,0,3,1,0,2,1,2,3,0,3,1,1,3,1,2,3,0,3,1,2,0,1,2,3,0,3,1,3,1,2,1,0,3,0,2,0,2,2,1,0,3,0,2,1,3,2,1,0,3,0,2,2,0,2,1,0,3,0,2,3,1,2,1,0,3,1,3,0,2,2,1,0,3,1,3,1,3,2,1,0,3,1,3,2,0,2,1,0,3,1,3,3,1,2,1,0,3,2,0,0,2,2,1,0,3,2,0,1,3,2,1,0,3,2,0,2,0,2,1,0,3,2,0,3,1,2,1,0,3,3,1,0,2,2,1,0,3,3,1,1,3,2,1,0,3,3,1,2,0,2,1,0,3,3,1,3,1,2,1,1,2,0,2,0,2,2,1,1,2,0,2,1,3,2,1,1,2,0,2,2,0,2,1,1,2,0,2,3,1,2,1,1,2,1,3,0,2,2,1,1,2,1,3,1,3,2,1,1,2,1,3,2,0,2,1,1,2,1,3,3,1,2,1,1,2,2,0,0,2,2,1,1,2,2,0,1,3,2,1,1,2,2,0,2,0,2,1,1,2,2,0,3,1,2,1,1,2,3,1,0,2,2,1,1,2,3,1,1,3,2,1,1,2,3,1,2,0,2,1,1,2,3,1,3,1,2,1,2,1,0,2,0,2,2,1,2,1,0,2,1,3,2,1,2,1,0,2,2,0,2,1,2,1,0,2,3,1,2,1,2,1,1,3,0,2,2,1,2,1,1,3,1,3,2,1,2,1,1,3,2,0,2,1,2,1,1,3,3,1,2,1,2,1,2,0,0,2,2,1,2,1,2,0,1,3,2,1,2,1,2,0,2,0,2,1,2,1,2,0,3,1,2,1,2,1,3,1,0,2,2,1,2,1,3,1,1,3,2,1,2,1,3,1,2,0,2,1,2,1,3,1,3,1,2,1,3,0,0,2,0,2,2,1,3,0,0,2,1,3,2,1,3,0,0,2,2,0,2,1,3,0,0,2,3,1,2,1,3,0,1,3,0,2,2,1,3,0,1,3,1,3,2,1,3,0,1,3,2,0,2,1,3,0,1,3,3,1,2,1,3,0,2,0,0,2,2,1,3,0,2,0,1,3,2,1,3,0,2,0,2,0,2,1,3,0,2,0,3,1,2,1,3,0,3,1,0,2,2,1,3,0,3,1,1,3,2,1,3,0,3,1,2,0,2,1,3,0,3,1,3,1,3,0,0,3,0,2,0,2,3,0,0,3,0,2,1,3,3,0,0,3,0,2,2,0,3,0,0,3,0,2,3,1,3,0,0,3,1,3,0,2,3,0,0,3,1,3,1,3,3,0,0,3,1,3,2,0,3,0,0,3,1,3,3,1,3,0,0,3,2,0,0,2,3,0,0,3,2,0,1,3,3,0,0,3,2,0,2,0,3,0,0,3,2,0,3,1,3,0,0,3,3,1,0,2,3,0,0,3,3,1,1,3,3,0,0,3,3,1,2,0,3,0,0,3,3,1,3,1,3,0,1,2,0,2,0,2,3,0,1,2,0,2,1,3,3,0,1,2,0,2,2,0,3,0,1,2,0,2,3,1,3,0,1,2,1,3,0,2,3,0,1,2,1,3,1,3,3,0,1,2,1,3,2,0,3,0,1,2,1,3,3,1,3,0,1,2,2,0,0,2,3,0,1,2,2,0,1,3,3,0,1,2,2,0,2,0,3,0,1,2,2,0,3,1,3,0,1,2,3,1,0,2,3,0,1,2,3,1,1,3,3,0,1,2,3,1,2,0,3,0,1,2,3,1,3,1,3,0,2,1,0,2,0,2,3,0,2,1,0,2,1,3,3,0,2,1,0,2,2,0,3,0,2,1,0,2,3,1,3,0,2,1,1,3,0,2,3,0,2,1,1,3,1,3,3,0,2,1,1,3,2,0,3,0,2,1,1,3,3,1,3,0,2,1,2,0,0,2,3,0,2,1,2,0,1,3,3,0,2,1,2,0,2,0,3,0,2,1,2,0,3,1,3,0,2,1,3,1,0,2,3,0,2,1,3,1,1,3,3,0,2,1,3,1,2,0,3,0,2,1,3,1,3,1,3,0,3,0,0,2,0,2,3,0,3,0,0,2,1,3,3,0,3,0,0,2,2,0,3,0,3,0,0,2,3,1,3,0,3,0,1,3,0,2,3,0,3,0,1,3,1,3,3,0,3,0,1,3,2,0,3,0,3,0,1,3,3,1,3,0,3,0,2,0,0,2,3,0,3,0,2,0,1,3,3,0,3,0,2,0,2,0,3,0,3,0,2,0,3,1,3,0,3,0,3,1,0,2,3,0,3,0,3,1,1,3,3,0,3,0,3,1,2,0,3,0,3,0,3,1,3,1}; static const __device__ float RTR_values[256] = {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}; diff --git a/lib/kernels/PLEGMA_baryons_deltas.cu b/lib/kernels/PLEGMA_baryons_deltas.cu index 2c9e799f..c3aacc39 100644 --- a/lib/kernels/PLEGMA_baryons_deltas.cu +++ b/lib/kernels/PLEGMA_baryons_deltas.cu @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" static const __device__ short int deltas_indices[3][16][4] = {0,0,0,0,0,0,1,1,0,0,2,2,0,0,3,3,1,1,0,0,1,1,1,1,1,1,2,2,1,1,3,3,2,2,0,0,2,2,1,1,2,2,2,2,2,2,3,3,3,3,0,0,3,3,1,1,3,3,2,2,3,3,3,3,0,0,0,0,0,0,1,1,0,0,2,2,0,0,3,3,1,1,0,0,1,1,1,1,1,1,2,2,1,1,3,3,2,2,0,0,2,2,1,1,2,2,2,2,2,2,3,3,3,3,0,0,3,3,1,1,3,3,2,2,3,3,3,3,0,1,0,1,0,1,1,0,0,1,2,3,0,1,3,2,1,0,0,1,1,0,1,0,1,0,2,3,1,0,3,2,2,3,0,1,2,3,1,0,2,3,2,3,2,3,3,2,3,2,0,1,3,2,1,0,3,2,2,3,3,2,3,2}; static const __device__ float deltas_values[3][16] = {1,-1,-1,1,-1,1,1,-1,-1,1,1,-1,1,-1,-1,1,1,1,-1,-1,1,1,-1,-1,-1,-1,1,1,-1,-1,1,1,1,1,-1,-1,1,1,-1,-1,-1,-1,1,1,-1,-1,1,1}; diff --git a/lib/kernels/PLEGMA_baryons_udsc.cu b/lib/kernels/PLEGMA_baryons_udsc.cu index 135159c9..f0876fcb 100644 --- a/lib/kernels/PLEGMA_baryons_udsc.cu +++ b/lib/kernels/PLEGMA_baryons_udsc.cu @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_baryons_udsc.cuh" template __global__ void create_prop_product(genericTex *propProd, @@ -162,19 +162,19 @@ void contract_baryons_udsc_host(ProfileStruct &ps, Float2 *d_partial_block = NULL; size_t alloc_size = (runFT==true) ? (volume * (ps.tp.grid.x/time_step)):volume; hostMalloc(h_partial_block, alloc_size * sizeof(Float2)); - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); short *idxs; Float2 *vals; int size = 0; for(int j=0; j)); + idxs=(short*)device_malloc(6*size*sizeof(short)); + vals=(Float2*)device_malloc(size*sizeof(Float2)); int shift = 0; for(int j=0; j), cudaMemcpyHostToDevice); + qudaMemcpy(idxs+6*shift, BP_prop_prods_idxs[i][j], 6*BP_prop_prods_count[i][j]*sizeof(short), qudaMemcpyHostToDevice); + qudaMemcpy(vals+shift, BP_prop_prods_vals[i][j], BP_prop_prods_count[i][j]*sizeof(Float2), qudaMemcpyHostToDevice); shift += BP_prop_prods_count[i][j]; } @@ -183,19 +183,19 @@ void contract_baryons_udsc_host(ProfileStruct &ps, PLEGMA_Field3D *propProd[time_step]; genericTex *texPropProd = NULL; if (ps.tp.aux.x == 2) { - cudaMalloc((void**)&texPropProd, time_step * sizeof(genericTex) ); + texPropProd=(genericTex*)device_malloc( time_step * sizeof(genericTex) ); for(int t=0; t(DEVICE, N_SPINS*N_SPINS*N_SPINS*N_SPINS*N_SPINS*N_SPINS, NO_GHOSTS, false, false); } - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { goto exit; } for(int t=0; t(*(propProd[t]))); - cudaMemcpy(texPropProd+t, holder.back().get(), sizeof(genericTex), cudaMemcpyHostToDevice); + qudaMemcpy(texPropProd+t, holder.back().get(), sizeof(genericTex), qudaMemcpyHostToDevice); } } else { - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { goto exit; } } for(int it=0; it < t_size; it+=time_step) { @@ -226,7 +226,7 @@ void contract_baryons_udsc_host(ProfileStruct &ps, (*propTex1, *propTex2, *propTex3, d_partial_block, BP_prop_prods_count[i][j], idxs+6*shift, vals+shift, source, runFT, *moms, it, std::min(t_size-it, time_step), maxT); } - cudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), cudaMemcpyDeviceToHost); + qudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), qudaMemcpyDeviceToHost); if(runFT==true){ int accumX = ps.tp.grid.x/time_step; Float2 *reduction = result + (j*t_size + it)*volume3D; @@ -245,13 +245,13 @@ void contract_baryons_udsc_host(ProfileStruct &ps, } exit: hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); d_partial_block=NULL; + device_free(d_partial_block); d_partial_block=NULL; - cudaFree(idxs); - cudaFree(vals); + device_free(idxs); + device_free(vals); if (ps.tp.aux.x == 2) { - cudaFree(texPropProd); + device_free(texPropProd); holder.clear(); for(int t=0; t -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_bcud_tetraquarks.cuh" #include template @@ -113,8 +113,8 @@ void contract_tetraquarks_bcud_host(ProfileStruct &ps, shift += TETRA_bcud_prop_prods_count[i][j]; } - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { goto exit; } for(int it=0; it < t_size; it+=time_step) { @@ -151,11 +151,11 @@ void contract_tetraquarks_bcud_host(ProfileStruct &ps, } exit: hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); d_partial_block=NULL; + device_free(d_partial_block); d_partial_block=NULL; - cudaFree(idxs); - cudaFree(col_contr); - cudaFree(vals); + device_free(idxs); + device_free(col_contr); + device_free(vals); // if (ps.tp.aux.x == 2) { // cudaFree(texPropProd); diff --git a/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu b/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu index 1f170e77..5dced21d 100644 --- a/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu +++ b/lib/kernels/PLEGMA_bcud_tetraquarks_stochastic.cu @@ -123,26 +123,26 @@ void contract_tetraquarks_bcud_stochastic_host(ProfileStruct &ps, Float2 *d_partial_block = NULL; size_t alloc_size = (runFT==true) ? (volume * (ps.tp.grid.x/time_step)):volume; hostMalloc(h_partial_block, alloc_size * sizeof(Float2)); - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc( alloc_size * sizeof(Float2) ); short *idxs, *col_contr; Float2 *vals; int size = 0; for(int j=0; j)); + idxs=(short*)device_malloc( 8*size*sizeof(short)); + col_contr=(short*)device_malloc( 8*size*sizeof(short)); + vals=(Float2*)device_malloc( size*sizeof(Float2)); int shift = 0; for(int j=0; j), cudaMemcpyHostToDevice); + qudaMemcpy(idxs+8*shift, TETRA_bcud_stoch_prop_prods_idxs[i][j], 8*TETRA_bcud_stoch_prop_prods_count[i][j]*sizeof(short), qudaMemcpyHostToDevice); + qudaMemcpy(col_contr+8*shift, TETRA_bcud_stoch_prop_prods_col_contr[i][j], 8*TETRA_bcud_stoch_prop_prods_count[i][j]*sizeof(short), qudaMemcpyHostToDevice); + qudaMemcpy(vals+shift, TETRA_bcud_stoch_prop_prods_vals[i][j], TETRA_bcud_stoch_prop_prods_count[i][j]*sizeof(Float2), qudaMemcpyHostToDevice); shift += TETRA_bcud_stoch_prop_prods_count[i][j]; } - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess) { goto exit; } for(int it=0; it < t_size; it+=time_step) { @@ -158,7 +158,7 @@ void contract_tetraquarks_bcud_stochastic_host(ProfileStruct &ps, (*propTex1, *propTex2, *propTex3, *propTex4, *propTex5, *propTex6, *propTex7, *propTex8, d_partial_block, TETRA_bcud_stoch_prop_prods_count[i][j], idxs+8*shift, col_contr+8*shift, vals+shift, source, runFT, *moms, it, std::min(t_size-it, time_step), maxT); - cudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), cudaMemcpyDeviceToHost); + qudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), qudaMemcpyDeviceToHost); if(runFT==true){ int accumX = ps.tp.grid.x/time_step; Float2 *reduction = result + (j*t_size + it)*volume3D; @@ -177,11 +177,11 @@ void contract_tetraquarks_bcud_stochastic_host(ProfileStruct &ps, } exit: hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); d_partial_block=NULL; + device_free(d_partial_block); d_partial_block=NULL; - cudaFree(idxs); - cudaFree(col_contr); - cudaFree(vals); + device_free(idxs); + device_free(col_contr); + device_free(vals); // if (ps.tp.aux.x == 2) { // cudaFree(texPropProd); diff --git a/lib/kernels/PLEGMA_contractG5_bilinear.cuh b/lib/kernels/PLEGMA_contractG5_bilinear.cuh index 26037485..8937264c 100644 --- a/lib/kernels/PLEGMA_contractG5_bilinear.cuh +++ b/lib/kernels/PLEGMA_contractG5_bilinear.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template diff --git a/lib/kernels/PLEGMA_covD.cuh b/lib/kernels/PLEGMA_covD.cuh index 0f776272..04059933 100644 --- a/lib/kernels/PLEGMA_covD.cuh +++ b/lib/kernels/PLEGMA_covD.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" using namespace plegma; template __global__ void covD_kernel(vector2 out, @@ -16,13 +16,13 @@ __global__ void covD_kernel(vector2 out, if(dirOr < 4){ gTex.get(G,dir,sid); - vTex.get(Sin,sid,dir); + vTex.template get(Sin,sid,dir); mul_G_V(Sout,G,Sin); out.set(Sout,sid); } else{ - gTex.get(G,dir,sid,dir); - vTex.get(Sin,sid,dir); + gTex.template get(G,dir,sid,dir); + vTex.template get(Sin,sid,dir); mul_Gdag_V(Sout,G,Sin); out.set(Sout,sid); } diff --git a/lib/kernels/PLEGMA_field_utils.cuh b/lib/kernels/PLEGMA_field_utils.cuh index 6f1b1c50..001a84a5 100644 --- a/lib/kernels/PLEGMA_field_utils.cuh +++ b/lib/kernels/PLEGMA_field_utils.cuh @@ -1,7 +1,14 @@ +#pragma once +#if defined (__NVCC__) #include +#endif +#if defined (__HIP__) +#include +#endif + #include #include -#include +#include "PLEGMA_kernel_utils.cuh" #include #include #include @@ -195,7 +202,7 @@ __inline__ __device__ Float2 rootsunity<4>(int order){ } template -__global__ void genStochasticUniform_kernel(cuRNGState *state, int length_field, Float *inout, bool is4D){ +__global__ void genStochasticUniform_kernel(RNGState *state, int length_field, Float *inout, bool is4D){ Float2 *inout2 = (Float2 *) inout; int sid = blockIdx.x*blockDim.x + threadIdx.x; @@ -225,7 +232,7 @@ void set_stochastic( PLEGMA_RNG &rng_state, PLEGMA_Field &inOut, int fiel genStochasticUniform_kernel<<>>(rng_state.State(), field_deg_free, inOut.D_elem(), inOut.is4D()); } template -__global__ void genRandomUniform_kernel(cuRNGState *state, int length_field, Float *inout){ +__global__ void genRandomUniform_kernel(RNGState *state, int length_field, Float *inout){ Float2 *inout2 = (Float2 *) inout; int sid = blockIdx.x*blockDim.x + threadIdx.x; @@ -235,7 +242,7 @@ __global__ void genRandomUniform_kernel(cuRNGState *state, int length_field, Flo } template -__global__ void genRandomNormal_kernel(cuRNGState *state, int length_field, Float *inout){ +__global__ void genRandomNormal_kernel(RNGState *state, int length_field, Float *inout){ Float2 *inout2 = (Float2 *) inout; int sid = blockIdx.x*blockDim.x + threadIdx.x; @@ -260,7 +267,7 @@ void set_random( PLEGMA_RNG &rng_state, PLEGMA_Field &inOut, int field_de template struct HadCol{ int ih; - __device__ HadCol(int ih):ih(ih){} + __device__ __host__ HadCol(int ih):ih(ih){} __device__ int HadamardElements(int i, int j){ int sum=0; for(int k = 0 ; k < 32 ; k++){ @@ -341,8 +348,8 @@ static void __global__ trPmunu_kernel(FloatA *out, gauge2 u, int mu, int x|-->--| **/ // U_\mu(x) * U_\nu(x+\mu) * U^dag_\mu(x+nu) * U^\dag_\nu(x) - u.get(U1,mu,sid); u.get(U2,nu,sid,mu); mul_G_G(U3,U1,U2); - u.get(U2,mu,sid,nu); mul_G_Gdag(U1,U3,U2); + u.get(U1,mu,sid); u.template get(U2,nu,sid,mu); mul_G_G(U3,U1,U2); + u.template get(U2,mu,sid,nu); mul_G_Gdag(U1,U3,U2); u.get(U2,nu,sid); mul_G_Gdag(U3,U1,U2); out2[sid]= U3[0][0] + U3[1][1] + U3[2][2]; } diff --git a/lib/kernels/PLEGMA_fmunu_utils.cuh b/lib/kernels/PLEGMA_fmunu_utils.cuh index 54e82753..71f6123e 100644 --- a/lib/kernels/PLEGMA_fmunu_utils.cuh +++ b/lib/kernels/PLEGMA_fmunu_utils.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" using namespace plegma; template @@ -20,8 +20,8 @@ __device__ void clover_leaves(Float2 F[N_COLS][N_COLS], gauge2 &u, x|-->--| **/ // U_\mu(x) * U_\nu(x+\mu) * U^dag_\mu(x+nu) * U^\dag_\nu(x) - u.get(U1,mu,sid); u.get(U2,nu,sid,mu); mul_G_G(P,U1,U2); - u.get(U2,mu,sid,nu); mul_G_Gdag(U1,P,U2); + u.get(U1,mu,sid); u.template get(U2,nu,sid,mu); mul_G_G(P,U1,U2); + u.template get(U2,mu,sid,nu); mul_G_Gdag(U1,P,U2); u.get(U2,nu,sid); mul_G_Gdag(P,U1,U2); G_plus_aG( F, P, 1.); @@ -33,9 +33,9 @@ __device__ void clover_leaves(Float2 F[N_COLS][N_COLS], gauge2 &u, |-->--|x **/ // U_\nu(x) * U^dag_\mu(x-mu+nu) * U^dag_\nu(x-mu) * U_\mu(x-mu) - u.get(U1,nu,sid); u.get(U2,mu,sid,mu,nu); mul_G_Gdag(P,U1,U2); - u.get(U2,nu,sid,mu); mul_G_Gdag(U1,P,U2); - u.get(U2,mu,sid,mu); mul_G_G(P,U1,U2); + u.get(U1,nu,sid); u.template get(U2,mu,sid,mu,nu); mul_G_Gdag(P,U1,U2); + u.template get(U2,nu,sid,mu); mul_G_Gdag(U1,P,U2); + u.template get(U2,mu,sid,mu); mul_G_G(P,U1,U2); G_plus_aG( F, P, 1.); @@ -46,9 +46,9 @@ __device__ void clover_leaves(Float2 F[N_COLS][N_COLS], gauge2 &u, |-->--| **/ //U^\dag_\mu(x-mu) * U^\dag_\nu(x-nu-mu) * U_\mu(x-nu-mu) * U_\nu(x-nu) - u.get(U1,mu,sid,mu); u.get(U2,nu,sid,nu,mu); mul_Gdag_Gdag(P,U1,U2); - u.get(U2,mu,sid,nu,mu); mul_G_G(U1,P,U2); - u.get(U2,nu,sid,nu); mul_G_G(P,U1,U2); + u.template get(U1,mu,sid,mu); u.template get(U2,nu,sid,nu,mu); mul_Gdag_Gdag(P,U1,U2); + u.template get(U2,mu,sid,nu,mu); mul_G_G(U1,P,U2); + u.template get(U2,nu,sid,nu); mul_G_G(P,U1,U2); G_plus_aG( F, P, 1.); /** @@ -58,8 +58,8 @@ __device__ void clover_leaves(Float2 F[N_COLS][N_COLS], gauge2 &u, |-->--| **/ //U^\dag_\nu(x-nu) * U_\mu(x-nu) * U_\nu(x+mu-nu) * U^\dag_\mu(x) - u.get(U1,nu,sid,nu); u.get(U2,mu,sid,nu); mul_Gdag_G(P,U1,U2); - u.get(U2,nu,sid,mu,nu); mul_G_G(U1,P,U2); + u.template get(U1,nu,sid,nu); u.template get(U2,mu,sid,nu); mul_Gdag_G(P,U1,U2); + u.template get(U2,nu,sid,mu,nu); mul_G_G(U1,P,U2); u.get(U2,mu,sid); mul_G_Gdag(P,U1,U2); G_plus_aG( F, P, 1.); diff --git a/lib/kernels/PLEGMA_gFixing.cuh b/lib/kernels/PLEGMA_gFixing.cuh index 73e4d9ea..39fb37f1 100644 --- a/lib/kernels/PLEGMA_gFixing.cuh +++ b/lib/kernels/PLEGMA_gFixing.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_SU3_projection.cuh" template static __global__ void gluonField_kernel(gauge2 Rout, gauge2 Rin){ @@ -45,7 +45,7 @@ static __global__ void gTransformLandau_kernel(su3_2 Rg, gauge2 RA for(int mu = 0; mu < N_DIMS; mu++){ RA.get(AS,mu,sid); G_plus_aG(gS,AS,1.); - RA.get(AS,mu,sid,mu); + RA.template get(AS,mu,sid,mu); Gdag(AS); G_plus_aG(gS,AS,1.); } @@ -82,7 +82,7 @@ static __global__ void gTransformMulALandau_kernel(su3_2 Rg, gauge2(gS,sid,mu); + Rg.template get(gS,sid,mu); mul_G_Gdag(tmp,AS,gS); RA.set(tmp,mu,sid); } diff --git a/lib/kernels/PLEGMA_gammas.cuh b/lib/kernels/PLEGMA_gammas.cuh index 978f802b..21b4a68c 100644 --- a/lib/kernels/PLEGMA_gammas.cuh +++ b/lib/kernels/PLEGMA_gammas.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_complex.cuh" #ifndef PLEGMA_GAMMAS_CUH #define PLEGMA_GAMMAS_CUH diff --git a/lib/kernels/PLEGMA_gammas_scatt.cuh b/lib/kernels/PLEGMA_gammas_scatt.cuh index 4872c77e..8da53e3d 100644 --- a/lib/kernels/PLEGMA_gammas_scatt.cuh +++ b/lib/kernels/PLEGMA_gammas_scatt.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_complex.cuh" #ifndef PLEGMA_GAMMAS_SCATT_CUH #define PLEGMA_GAMMAS_SCATT_CUH diff --git a/lib/kernels/PLEGMA_gauge_utils.cuh b/lib/kernels/PLEGMA_gauge_utils.cuh index 02f23af7..9939101b 100644 --- a/lib/kernels/PLEGMA_gauge_utils.cuh +++ b/lib/kernels/PLEGMA_gauge_utils.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" using namespace plegma; template< typename Float> @@ -51,9 +51,9 @@ static void scale_dir_wise(gauge2 gauge, Float* scale){ dim3 blockDim( THREADS_PER_BLOCK , 1, 1); dim3 gridDim( (gauge.volume() + blockDim.x -1)/blockDim.x , 1 , 1); Float2 *d_scale; - cudaMalloc((void**) &d_scale, N_DIMS*sizeof(Float2)); - cudaMemcpy( d_scale, scale, N_DIMS*sizeof(Float2),cudaMemcpyHostToDevice); + d_scale=(Float2*)device_malloc(N_DIMS*sizeof(Float2)); + qudaMemcpy( d_scale, scale, N_DIMS*sizeof(Float2),qudaMemcpyHostToDevice); scale_dir_wise_kernel<<>>(gauge, d_scale); - cudaFree(d_scale); + device_free(d_scale); checkQudaError(); } diff --git a/lib/kernels/PLEGMA_gaussian_smearing.cuh b/lib/kernels/PLEGMA_gaussian_smearing.cuh index 30c4c2eb..d203758d 100644 --- a/lib/kernels/PLEGMA_gaussian_smearing.cuh +++ b/lib/kernels/PLEGMA_gaussian_smearing.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template __global__ void gaussian_smearing_kernel(vectorTexout, @@ -21,14 +21,14 @@ __global__ void gaussian_smearing_kernel(vectorTexout, // we don't smear the time -> N_DIMS-1 #pragma unroll for(int dir = 0; dir < N_DIMS-1; dir++) { - vecInTex.get(S, sid, dir); + vecInTex.template get(S, sid, dir); if(isNotZeroV(S)) { gaugeTex.get(G, dir, sid); mul_G_V(tmp,G,S); } - vecInTex.get(S, sid, dir); + vecInTex.template get(S, sid, dir); if(isNotZeroV(S)) { - gaugeTex.get(G, dir, sid, dir); + gaugeTex.template get(G, dir, sid, dir); mul_Gdag_V(tmp,G,S); } } @@ -77,15 +77,15 @@ __global__ void gaussian_smearing_only_ghost_kernel(vectorTexout, tmp[mu][c] = 0.; if(sign==DIR_PLUS) { - vecInTex.get(S, sid, dir); + vecInTex.template get(S, sid, dir); if(isNotZeroV(S)) { gaugeTex.get(G, dir, sid); mul_G_V(tmp,G,S); } } else { - vecInTex.get(S, sid, dir); + vecInTex.template get(S, sid, dir); if(isNotZeroV(S)) { - gaugeTex.get(G, dir, sid, dir); + gaugeTex.template get(G, dir, sid, dir); mul_Gdag_V(tmp,G,S); } } diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu index 2b5717c8..1def2abe 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks.cu @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_heavy_light_tetraquarks.cuh" #include #include @@ -114,8 +114,8 @@ void contract_tetraquarks_host(ProfileStruct &ps, shift += TETRA_prop_prods_count[i][j]; } - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { goto exit; } for(int it=0; it < t_size; it+=time_step) { @@ -152,11 +152,11 @@ void contract_tetraquarks_host(ProfileStruct &ps, } exit: hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); d_partial_block=NULL; + device_free(d_partial_block); d_partial_block=NULL; - cudaFree(idxs); - cudaFree(col_contr); - cudaFree(vals); + device_free(idxs); + device_free(col_contr); + device_free(vals); // if (ps.tp.aux.x == 2) { // cudaFree(texPropProd); diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu index 537b8654..25618051 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks_open_contractions.cu @@ -1,8 +1,8 @@ #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_heavy_light_tetraquarks.cuh" #include #include using namespace plegma; @@ -122,8 +122,8 @@ static void tetraquark_open_index_host(ProfileStruct &ps, Float2 *result auto propTex1 = toTexture(prop1); auto propTex2 = toTexture(prop2); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { int t_step = std::min(t_size-it, time_step); dim3 grid = ps.tp.grid; @@ -131,10 +131,10 @@ static void tetraquark_open_index_host(ProfileStruct &ps, Float2 *result tetraquark_open_index_device <<>> (d_partial_block, *propTex1, *propTex2, listGammas, it, t_step, maxT, source, runFT, *moms, s1); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; qudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu b/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu index eab9c4af..e8166293 100644 --- a/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu +++ b/lib/kernels/PLEGMA_heavy_light_tetraquarks_stochastic.cu @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_heavy_light_tetraquarks.cuh" template @@ -123,26 +123,26 @@ void contract_tetraquarks_stochastic_host(ProfileStruct &ps, Float2 *d_partial_block = NULL; size_t alloc_size = (runFT==true) ? (volume * (ps.tp.grid.x/time_step)):volume; hostMalloc(h_partial_block, alloc_size * sizeof(Float2)); - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); short *idxs, *col_contr; Float2 *vals; int size = 0; for(int j=0; j)); + idxs=(short*)device_malloc( 8*size*sizeof(short)); + col_contr=(short*)device_malloc( 8*size*sizeof(short)); + vals=(Float2*)device_malloc(size*sizeof(Float2)); int shift = 0; for(int j=0; j), cudaMemcpyHostToDevice); + qudaMemcpy(idxs+8*shift, TETRA_stoch_prop_prods_idxs[i][j], 8*TETRA_stoch_prop_prods_count[i][j]*sizeof(short), qudaMemcpyHostToDevice); + qudaMemcpy(col_contr+8*shift, TETRA_stoch_prop_prods_col_contr[i][j], 8*TETRA_stoch_prop_prods_count[i][j]*sizeof(short), qudaMemcpyHostToDevice); + qudaMemcpy(vals+shift, TETRA_stoch_prop_prods_vals[i][j], TETRA_stoch_prop_prods_count[i][j]*sizeof(Float2), qudaMemcpyHostToDevice); shift += TETRA_stoch_prop_prods_count[i][j]; } - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { goto exit; } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { goto exit; } for(int it=0; it < t_size; it+=time_step) { @@ -158,7 +158,7 @@ void contract_tetraquarks_stochastic_host(ProfileStruct &ps, (*propTex1, *propTex2, *propTex3, *propTex4, *propTex5, *propTex6, *propTex7, *propTex8, d_partial_block, TETRA_stoch_prop_prods_count[i][j], idxs+8*shift, col_contr+8*shift, vals+shift, source, runFT, *moms, it, std::min(t_size-it, time_step), maxT); - cudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), cudaMemcpyDeviceToHost); + qudaMemcpy(h_partial_block , d_partial_block , alloc_size*sizeof(Float2), qudaMemcpyDeviceToHost); if(runFT==true){ int accumX = ps.tp.grid.x/time_step; Float2 *reduction = result + (j*t_size + it)*volume3D; @@ -177,11 +177,11 @@ void contract_tetraquarks_stochastic_host(ProfileStruct &ps, } exit: hostFree(h_partial_block, alloc_size*sizeof(Float2)); - cudaFree(d_partial_block); d_partial_block=NULL; + device_free(d_partial_block); d_partial_block=NULL; - cudaFree(idxs); - cudaFree(col_contr); - cudaFree(vals); + device_free(idxs); + device_free(col_contr); + device_free(vals); // if (ps.tp.aux.x == 2) { // cudaFree(texPropProd); diff --git a/lib/kernels/PLEGMA_kernel_getSet.cuh b/lib/kernels/PLEGMA_kernel_getSet.cuh index 32c165a3..b022e7b5 100644 --- a/lib/kernels/PLEGMA_kernel_getSet.cuh +++ b/lib/kernels/PLEGMA_kernel_getSet.cuh @@ -83,7 +83,7 @@ namespace plegma { size_t stride; inline __host__ __device__ size_t volume() const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__ ) || ( __HIP__ ) return is4D ? DGC_localVolume : DGC_localVolume3D; #else return is4D ? HGC_localVolume : HGC_localVolume3D; @@ -91,7 +91,7 @@ namespace plegma { } inline __host__ __device__ size_t sideGhostVolume() const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_sideGhostVolume : DGC_sideGhostVolume3D; #else return is4D ? HGC_sideGhostVolume : HGC_sideGhostVolume3D; @@ -99,7 +99,7 @@ namespace plegma { } inline __host__ __device__ size_t sideGhostL(const short& dir) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_surface3D[dir] : (DGC_surface3D[dir]/DGC_localL[DIM_T]); #else return is4D ? HGC_surface3D[dir] : (HGC_surface3D[dir]/HGC_localL[DIM_T]); @@ -107,7 +107,7 @@ namespace plegma { } inline __host__ __device__ size_t sideGhostShift(const short& dir, const ORIENTATION& sign) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_sideGhost[dir][sign] : (DGC_sideGhost[dir][sign]/DGC_localL[DIM_T]); #else return is4D ? HGC_sideGhost[dir][sign] : (HGC_sideGhost[dir][sign]/HGC_localL[DIM_T]); @@ -115,7 +115,7 @@ namespace plegma { } inline __host__ __device__ size_t cornerGhostVolume() const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_cornerGhostVolume : DGC_cornerGhostVolume3D; #else return is4D ? HGC_cornerGhostVolume : HGC_cornerGhostVolume3D; @@ -123,7 +123,7 @@ namespace plegma { } inline __host__ __device__ size_t cornerGhostL(const short& dir1, const short& dir2) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_surface2D[OFF2(dir1,dir2)] : (DGC_surface2D[OFF2(dir1,dir2)]/DGC_localL[DIM_T]); #else return is4D ? HGC_surface2D[OFF2(dir1,dir2)] : (HGC_surface2D[OFF2(dir1,dir2)]/HGC_localL[DIM_T]); @@ -131,7 +131,7 @@ namespace plegma { } inline __host__ __device__ size_t vertexGhostL(const short& dir1, const short& dir2, const short& dir3) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_surface1D[OFF3(dir1,dir2,dir3)] : (DGC_surface1D[OFF3(dir1,dir2,dir3)]/DGC_localL[DIM_T]); #else return is4D ? HGC_surface1D[OFF3(dir1,dir2,dir3)] : (HGC_surface1D[OFF3(dir1,dir2,dir3)]/HGC_localL[DIM_T]); @@ -140,7 +140,7 @@ namespace plegma { inline __host__ __device__ size_t cornerGhostShift(const short& dir1, const short& dir2, const ORIENTATION& sign1, const ORIENTATION& sign2) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_cornerGhost[OFF2SIGN(dir1,dir2,sign1,sign2)] : (DGC_cornerGhost[OFF2SIGN(dir1,dir2,sign1,sign2)]/DGC_localL[DIM_T]); #else return is4D ? HGC_cornerGhost[OFF2SIGN(dir1,dir2,sign1,sign2)] : (HGC_cornerGhost[OFF2SIGN(dir1,dir2,sign1,sign2)]/HGC_localL[DIM_T]); @@ -149,7 +149,7 @@ namespace plegma { inline __host__ __device__ size_t vertexGhostShift(const short& dir1, const short& dir2, const short& dir3, const ORIENTATION& sign1, const ORIENTATION& sign2, const ORIENTATION& sign3) const { - #ifdef __CUDA_ARCH__ + #if defined ( __CUDA_ARCH__) || ( __HIP__ ) return is4D ? DGC_vertexGhost[OFF3SIGN(dir1,dir2,dir3,sign1,sign2,sign3)] : (DGC_vertexGhost[OFF3SIGN(dir1,dir2,dir3,sign1,sign2,sign3)]/DGC_localL[DIM_T]); #else return is4D ? HGC_vertexGhost[OFF3SIGN(dir1,dir2,dir3,sign1,sign2,sign3)] : (HGC_vertexGhost[OFF3SIGN(dir1,dir2,dir3,sign1,sign2,sign3)]/HGC_localL[DIM_T]); @@ -487,11 +487,12 @@ namespace plegma { template struct texture : pFloat2 { - cudaTextureObject_t tex; - - __host__ __device__ texture(cudaTextureObject_t tex, Float2* p, int site_size, bool is4D, bool changeValue) : - pFloat2(p, site_size, is4D, changeValue), tex(tex) { } +// cudaTextureObject_t tex; + __host__ __device__ texture(//cudaTextureObject_t tex, + Float2* p, int site_size, bool is4D, bool changeValue) : + pFloat2(p, site_size, is4D, changeValue) { } +/* #ifdef PLEGMA_TEXTURE // Fetch is going to be specialized after inline __device__ Float2 fetch(const size_t& i) const; @@ -499,8 +500,9 @@ namespace plegma { return this->returnZero ? Float2(0) : texture::fetch(i*this->stride + this->sid); } #endif + */ }; - +/* #ifdef PLEGMA_TEXTURE // Here we specialize fetch template<> inline __device__ Float2 texture::fetch(const size_t& i) const { @@ -511,6 +513,7 @@ namespace plegma { return (Float2) make_double2(__hiloint2double(v.y, v.x), __hiloint2double(v.w, v.z)); } #endif +*/ template struct generic : T { @@ -860,12 +863,12 @@ namespace plegma { template class T, template class Tfield, typename Float> static inline std::shared_ptr> toTexture(const Tfield& field) { - return std::shared_ptr>(new T(field.createTexObject(), (Float2*) field.D_elem(), field.Field_length(), field.is4D(), true), [&](T* ptr){field.destroyTexObject(ptr->tex); delete ptr;}); + return std::shared_ptr>(new T((Float2*) field.D_elem(), field.Field_length(), field.is4D(), true), [&](T* ptr){delete ptr;}); } template class Tfield, typename Float> static inline std::shared_ptr toTexture(const Tfield& field) { - return std::shared_ptr(new T(field.createTexObject(), (Float2*) field.D_elem(), field.Field_length(), field.is4D(), true), [&](T* ptr){field.destroyTexObject(ptr->tex); delete ptr;}); + return std::shared_ptr(new T( (Float2*) field.D_elem(), field.Field_length(), field.is4D(), true), [&](T* ptr){delete ptr;}); } diff --git a/lib/kernels/PLEGMA_kernel_tuner.cuh b/lib/kernels/PLEGMA_kernel_tuner.cuh index 67b3c810..efa0351a 100644 --- a/lib/kernels/PLEGMA_kernel_tuner.cuh +++ b/lib/kernels/PLEGMA_kernel_tuner.cuh @@ -2,7 +2,13 @@ #include #include #include + +#ifdef __NVCC__ #include +#elif __HIP__ +#include +#endif + using namespace quda; #ifndef PLEGMA_KERNEL_TUNER_H @@ -11,7 +17,7 @@ using namespace quda; #define THREADS_PER_BLOCK 64 -extern __device__ cudaDeviceProp devProp; +//extern __device__ cudaDeviceProp devProp; // struct that contains all variables // necessary for the tuning evaluation @@ -49,14 +55,14 @@ struct ProfileStruct{ }; template -struct seq { }; +struct sequ { }; template struct gens : gens { }; template struct gens<0, S...> { - typedef seq type; + typedef sequ type; }; // class to perform the kernel tuning @@ -171,12 +177,16 @@ protected: // launching utilities template - void callKernel(TuneParam tp, const qudaStream_t stream, seq) { + void callKernel(TuneParam tp, const qudaStream_t stream, sequ) { if( typeid(ProfileStruct &)==typeid(std::get<0>(args))) { // in case ProfileStruct is the first argument we call it as a function (*kernel)(std::get(args)...); } else { - (*kernel)<<>>(std::get(args)...); + #ifdef __NVCC__ + (*kernel)<<>>(std::get(args)...); + #elif __HIP__ + (*kernel)<<>>(std::get(args)...); + #endif } // cudaDeviceSynchronize(); } diff --git a/lib/kernels/PLEGMA_kernel_utils.cuh b/lib/kernels/PLEGMA_kernel_utils.cuh index d26d2eee..67f7d859 100644 --- a/lib/kernels/PLEGMA_kernel_utils.cuh +++ b/lib/kernels/PLEGMA_kernel_utils.cuh @@ -5,10 +5,10 @@ #include #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_complex.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_kernel_tuner.cuh" +#include "PLEGMA_gammas.cuh" #ifndef PLEGMA_KERNEL_UTILS_CUH #define PLEGMA_KERNEL_UTILS_CUH diff --git a/lib/kernels/PLEGMA_mesons.cuh b/lib/kernels/PLEGMA_mesons.cuh index 6190d411..db35dd37 100644 --- a/lib/kernels/PLEGMA_mesons.cuh +++ b/lib/kernels/PLEGMA_mesons.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" #include #pragma once using namespace plegma; @@ -23,7 +23,7 @@ __global__ void contract_mesons_device( propTex texProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[2*N_MESONS]; + Float2 accum[2*N_MESONS]; for(int i = 0 ; i < 2*N_MESONS ; i++){ accum[i] = 0.; } @@ -81,7 +81,7 @@ __global__ void contract_mesons_fourp_ultralocal_device( propTex texProp int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[16]; + Float2 accum[16]; for(int i = 0 ; i < 16 ; i++){ accum[i] = 0.; } @@ -160,7 +160,7 @@ __global__ void contract_mesons_fourp_ultralocal_oneendtrick_device( propTex=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[16]; + Float2 accum[16]; for(int i = 0 ; i < 16 ; i++){ accum[i] = 0.; } @@ -237,13 +237,13 @@ void contract_mesons_host( ProfileStruct &ps, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); + /*cudaError_t error=cudaPeekAtLastError(); if(error != cudaSuccess) { cudaFree(d_partial_block); return; - } + }*/ hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -254,10 +254,10 @@ void contract_mesons_host( ProfileStruct &ps, contract_mesons_device <<>> (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -275,9 +275,9 @@ void contract_mesons_host( ProfileStruct &ps, } } - printf("PLEGMA_mesons res %e %e %e t_size = %d, maxT = %d, source.w = %d, HGC_localVolume3D %d time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", result[0].norm2(),result[1].norm2(),result[2].norm(),t_size, maxT, source.w, HGC_localVolume3D, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); + //printf("PLEGMA_mesons res %e %e %e t_size = %d, maxT = %d, source.w = %d, HGC_localVolume3D %d time_step = %d, ps.tp.grid.x = %d, ps.tp.block.x = %d, ps.tp.shared_bytes = %d\n", result[0].norm2(),result[1].norm2(),result[2].norm(),t_size, maxT, source.w, HGC_localVolume3D, time_step, ps.tp.grid.x, ps.tp.block.x, ps.tp.shared_bytes); hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template @@ -303,13 +303,13 @@ void contract_mesons_fourp_ultralocal_host( ProfileStruct &ps, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { - cudaFree(d_partial_block); - return; - } +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess) { +// cudaFree(d_partial_block); +// return; +// } hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -323,10 +323,10 @@ void contract_mesons_fourp_ultralocal_host( ProfileStruct &ps, contract_mesons_fourp_ultralocal_device <<>> (*propTex1, *propTex2, *propTex3, *propTex4, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + // error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -344,7 +344,7 @@ void contract_mesons_fourp_ultralocal_host( ProfileStruct &ps, } } hostFree(h_partial_block, alloc_size*sizeof(FloatE)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template @@ -370,13 +370,13 @@ void contract_mesons_fourp_ultralocal_oneendtrick_host( ProfileStruct &ps, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); + /*cudaError_t error=cudaPeekAtLastError(); if(error != cudaSuccess) { cudaFree(d_partial_block); return; - } + }*/ hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -388,10 +388,10 @@ void contract_mesons_fourp_ultralocal_oneendtrick_host( ProfileStruct &ps, contract_mesons_fourp_ultralocal_oneendtrick_device <<>> (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -410,7 +410,7 @@ void contract_mesons_fourp_ultralocal_oneendtrick_host( ProfileStruct &ps, } hostFree(h_partial_block, alloc_size*sizeof(FloatE)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template diff --git a/lib/kernels/PLEGMA_mesonsAll.cuh b/lib/kernels/PLEGMA_mesonsAll.cuh index fc800c06..a207af02 100644 --- a/lib/kernels/PLEGMA_mesonsAll.cuh +++ b/lib/kernels/PLEGMA_mesonsAll.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_mesons.cuh" #include using namespace plegma; @@ -20,7 +20,7 @@ __global__ void contract_mesons_all_device( propTex texProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_PAIRS]; + Float2 accum[N_PAIRS]; for(int i = 0 ; i < N_PAIRS ; i++){ accum[i] = 0.; } @@ -119,14 +119,14 @@ void contract_mesons_all_host( ProfileStruct &ps, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; // cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); -d_partial_block=(Float2 *)device_malloc(alloc_size*sizeof(Float2)); + d_partial_block=(Float2 *)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { - cudaFree(d_partial_block); - return; - } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { + // cudaFree(d_partial_block); + // return; + //} hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -137,10 +137,10 @@ d_partial_block=(Float2 *)device_malloc(alloc_size*sizeof(Float2 contract_mesons_all_device <<>> (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -158,7 +158,7 @@ d_partial_block=(Float2 *)device_malloc(alloc_size*sizeof(Float2 } } hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template diff --git a/lib/kernels/PLEGMA_mesonsNew.cuh b/lib/kernels/PLEGMA_mesonsNew.cuh index cee69066..ba8dc460 100644 --- a/lib/kernels/PLEGMA_mesonsNew.cuh +++ b/lib/kernels/PLEGMA_mesonsNew.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_mesons.cuh" using namespace plegma; template @@ -17,7 +17,7 @@ __global__ void contract_mesons_new_device( propTex texProp1, int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_MESONS]; + Float2 accum[N_MESONS]; for(int i = 0 ; i < N_MESONS ; i++){ accum[i] = 0.; } @@ -82,13 +82,14 @@ void contract_mesons_new_host( ProfileStruct &ps, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); + d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); + //cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { - cudaFree(d_partial_block); - return; - } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { + // cudaFree(d_partial_block); + // return; + //} hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); @@ -99,10 +100,10 @@ void contract_mesons_new_host( ProfileStruct &ps, contract_mesons_new_device <<>> (*propTex1, *propTex2, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) break; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) break; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) break; if(runFT==true) { int accumX = ps.tp.grid.x/time_step; @@ -120,7 +121,7 @@ void contract_mesons_new_host( ProfileStruct &ps, } } hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template diff --git a/lib/kernels/PLEGMA_plaquette.cuh b/lib/kernels/PLEGMA_plaquette.cuh index 307e71dd..05403eb9 100644 --- a/lib/kernels/PLEGMA_plaquette.cuh +++ b/lib/kernels/PLEGMA_plaquette.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" #include using namespace plegma; @@ -18,8 +18,8 @@ static __global__ void calculatePlaquette_device(u1gaugeTex gTex, Float for(int dir1=0; dir1(G2,dir2,sid,dir1); - gTex.get(G3,dir1,sid,dir2); gTex.get(G4,dir2,sid); + gTex.get(G1,dir1,sid); gTex.template get(G2,dir2,sid,dir1); + gTex.template get(G3,dir1,sid,dir2); gTex.get(G4,dir2,sid); Float2 val = G1*G2*conj(G3)*conj(G4); // if(dir1 == 2 && dir2 == 3) // printf("%d %d %d %d %f %f\n",x[0],x[1],x[2],x[3],val.x,val.y); @@ -56,11 +56,11 @@ static __global__ void calculatePlaquette_device(gaugeTex gTex, Float *p for(int dir2=dir1+1; dir2(G2,dir2,sid,dir1); + gTex.template get(G2,dir2,sid,dir1); mul_G_G(G3,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 - gTex.get(G1,dir1,sid,dir2); + gTex.template get(G1,dir1,sid,dir2); gTex.get(G2,dir2,sid); mul_Gdag_Gdag(G4,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 @@ -84,14 +84,14 @@ static void calculatePlaquette_host(ProfileStruct& ps, TG gTex, Float& plaquette Float *d_partial_plaq = NULL; int gridDimX = ps.tp.grid.x; - cudaMalloc((void**)&d_partial_plaq, gridDimX * sizeof(Float)); + d_partial_plaq=(Float*)device_malloc( gridDimX * sizeof(Float)); calculatePlaquette_device<<>>(gTex, d_partial_plaq); Float *h_partial_plaq = NULL; hostMalloc(h_partial_plaq, gridDimX * sizeof(Float) ); if(h_partial_plaq == NULL) PLEGMA_error("Error allocate memory for host partial plaq"); - cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , cudaMemcpyDeviceToHost); - cudaFree(d_partial_plaq); + qudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , qudaMemcpyDeviceToHost); + device_free(d_partial_plaq); checkQudaError(); plaquette = 0.; diff --git a/lib/kernels/PLEGMA_plaquetteCorners.cuh b/lib/kernels/PLEGMA_plaquetteCorners.cuh index 47537c49..09e2571a 100644 --- a/lib/kernels/PLEGMA_plaquetteCorners.cuh +++ b/lib/kernels/PLEGMA_plaquetteCorners.cuh @@ -8,8 +8,8 @@ * Here the plaquette is computed 4 times by each site considering all the directions. */ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" using namespace plegma; @@ -33,11 +33,11 @@ static __global__ void calculatePlaquetteCorners_device(gaugeTex gaugeTe for(int dir2=dir1+1; dir2(G2,dir2,sid,dir1); + gaugeTex.template get(G2,dir2,sid,dir1); mul_G_G(G3,G1,G2); - gaugeTex.get(G1,dir1,sid,dir2); + gaugeTex.template get(G1,dir1,sid,dir2); gaugeTex.get(G2,dir2,sid); mul_Gdag_Gdag(G4,G1,G2); @@ -45,39 +45,39 @@ static __global__ void calculatePlaquetteCorners_device(gaugeTex gaugeTe trace += real_trace_mul_G_G(G3,G4); // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] - gaugeTex.get(G1,dir1,sid,dir1); + gaugeTex.template get(G1,dir1,sid,dir1); gaugeTex.get(G2,dir2,sid); mul_G_G(G3,G1,G2); - gaugeTex.get(G1,dir1,sid,dir1,dir2); - gaugeTex.get(G2,dir2,sid,dir1); + gaugeTex.template get(G1,dir1,sid,dir1,dir2); + gaugeTex.template get(G2,dir2,sid,dir1); mul_Gdag_Gdag(G4,G1,G2); trace += real_trace_mul_G_G(G3,G4); // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] - gaugeTex.get(G1,dir1,sid,dir1,dir2); - gaugeTex.get(G2,dir2,sid,dir2); + gaugeTex.template get(G1,dir1,sid,dir1,dir2); + gaugeTex.template get(G2,dir2,sid,dir2); mul_G_G(G3,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 - gaugeTex.get(G1,dir1,sid,dir1); - gaugeTex.get(G2,dir2,sid,dir1,dir2); + gaugeTex.template get(G1,dir1,sid,dir1); + gaugeTex.template get(G2,dir2,sid,dir1,dir2); mul_Gdag_Gdag(G4,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 trace += real_trace_mul_G_G(G3,G4); // flops = N_COLS*N_COLS*(2+1) // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] - gaugeTex.get(G1,dir1,sid,dir2); - gaugeTex.get(G2,dir2,sid,dir1,dir2); + gaugeTex.template get(G1,dir1,sid,dir2); + gaugeTex.template get(G2,dir2,sid,dir1,dir2); mul_G_G(G3,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 gaugeTex.get(G1,dir1,sid); - gaugeTex.get(G2,dir2,sid,dir2); + gaugeTex.template get(G2,dir2,sid,dir2); mul_Gdag_Gdag(G4,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 @@ -100,15 +100,15 @@ static void calculatePlaquetteCorners_host(ProfileStruct& ps, gaugeTex g Float *d_partial_plaq = NULL; int gridDimX = ps.tp.grid.x; - cudaMalloc((void**)&d_partial_plaq, gridDimX * sizeof(Float)); + d_partial_plaq=(Float*)device_malloc( gridDimX * sizeof(Float)); calculatePlaquetteCorners_device<<>>(gaugeTex, d_partial_plaq); Float *h_partial_plaq = NULL; hostMalloc(h_partial_plaq, gridDimX * sizeof(Float) ); if(h_partial_plaq == NULL) PLEGMA_error("Error allocate memory for host partial plaq"); - cudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , cudaMemcpyDeviceToHost); - cudaFree(d_partial_plaq); + qudaMemcpy(h_partial_plaq, d_partial_plaq , gridDimX * sizeof(Float) , qudaMemcpyDeviceToHost); + device_free(d_partial_plaq); checkQudaError(); plaquette = 0.; diff --git a/lib/kernels/PLEGMA_projectors.cuh b/lib/kernels/PLEGMA_projectors.cuh index 6c74b61c..e349ece4 100644 --- a/lib/kernels/PLEGMA_projectors.cuh +++ b/lib/kernels/PLEGMA_projectors.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_complex.cuh" #ifndef PLEGMA_PROJECTORS_CUH #define PLEGMA_PROJECTORS_CUH diff --git a/lib/kernels/PLEGMA_propagator_utils.cuh b/lib/kernels/PLEGMA_propagator_utils.cuh index e46aeece..e2744966 100644 --- a/lib/kernels/PLEGMA_propagator_utils.cuh +++ b/lib/kernels/PLEGMA_propagator_utils.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template diff --git a/lib/kernels/PLEGMA_scattreductions.cuh b/lib/kernels/PLEGMA_scattreductions.cuh index 7645ca51..d15d9cff 100644 --- a/lib/kernels/PLEGMA_scattreductions.cuh +++ b/lib/kernels/PLEGMA_scattreductions.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" #include #include using namespace plegma; @@ -380,12 +380,12 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator // cudaMalloc((void**)&d_partial_block, alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess) { - PLEGMA_printf("ERROR0\n"); - cudaFree(d_partial_block); - return; - } + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess) { + // PLEGMA_printf("ERROR0\n"); + // device_free(d_partial_block); + // return; + //} hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); KernelArr listGammas_i, listGammas_f; @@ -415,13 +415,13 @@ static void T_reductions_host( ProfileStruct &ps, TRED T, PLEGMA_ScattCorrelator T_kernels_wrapper(ps, T, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, *moms, listGammas_i, listGammas_f, S1, S2, S3 ); ps.tp.grid.x = grid.x; - cudaDeviceSynchronize(); + //cudaDeviceSynchronize(); - error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR1\n"); break;} + //error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR1\n"); break;} qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR2\n"); break;} + //error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR2\n"); break;} for(size_t tslicexmom = 0 ; tslicexmom< N_moms*std::min(t_size-it, time_step); tslicexmom++){ for(int f = 0 ; f < site_size; f++) { diff --git a/lib/kernels/PLEGMA_scattreductionsPiPi.cuh b/lib/kernels/PLEGMA_scattreductionsPiPi.cuh index a3a88dbc..7ed4db34 100644 --- a/lib/kernels/PLEGMA_scattreductionsPiPi.cuh +++ b/lib/kernels/PLEGMA_scattreductionsPiPi.cuh @@ -1,5 +1,5 @@ -#include -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" #include #include using namespace plegma; @@ -16,7 +16,7 @@ __global__ void PhixGxPhi_kernel( vectorTex vectorPhi0, KernelArr=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[16]; + Float2 accum[16]; for(int i = 0 ; i < 16 ; i++){ accum[i].x = 0.; accum[i].y = 0.; @@ -94,12 +94,12 @@ static void PhixGxPhi_host( ProfileStruct &ps, PLEGMA_ScattCorrelator d_partial_block=(Float2*)device_malloc(alloc_size*sizeof(Float2)); // Checking for allocation error. In case we return and let the tuner handle the error. - cudaError_t error=cudaPeekAtLastError(); + /*cudaError_t error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR0\n"); cudaFree(d_partial_block); return; - } + }*/ hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); KernelArr listGammas; @@ -121,13 +121,13 @@ static void PhixGxPhi_host( ProfileStruct &ps, PLEGMA_ScattCorrelator PhixGxPhi_kernel<<>>(*phiTex0, listGammas, *phiTex1, d_partial_block, it, std::min(t_size-it, time_step), maxT, source, *moms); - cudaDeviceSynchronize(); + //cudaDeviceSynchronize(); - error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR1\n"); break;} + //error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR1\n"); break;} qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*std::min(t_size-it, time_step)*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR2\n"); break;} +// error=cudaPeekAtLastError(); if(error != cudaSuccess) { PLEGMA_printf("ERROR2\n"); break;} for(size_t tslicexmom = 0 ; tslicexmom< N_moms*std::min(t_size-it, time_step); tslicexmom++){ for(int f = 0 ; f < site_size; f++) { diff --git a/lib/kernels/PLEGMA_scattreductionsT.cu b/lib/kernels/PLEGMA_scattreductionsT.cu index 2eaeb9c7..d05a7d61 100644 --- a/lib/kernels/PLEGMA_scattreductionsT.cu +++ b/lib/kernels/PLEGMA_scattreductionsT.cu @@ -1,8 +1,8 @@ -#include +#include "PLEGMA_kernel_utils.cuh" #include -#include -#include -#include +#include "PLEGMA_scattreductions.cuh" +#include "PLEGMA_scattreductionsT1.cuh" +#include "PLEGMA_scattreductionsT2.cuh" using namespace plegma; diff --git a/lib/kernels/PLEGMA_scattreductionsT1.cuh b/lib/kernels/PLEGMA_scattreductionsT1.cuh index 064d7802..0141545d 100644 --- a/lib/kernels/PLEGMA_scattreductionsT1.cuh +++ b/lib/kernels/PLEGMA_scattreductionsT1.cuh @@ -1,5 +1,5 @@ -#include -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -16,7 +16,7 @@ __global__ void T1_kernel( KernelArr listGammas_i, KernelArr accum[N_GAMMAS_SCATT_I*N_GAMMAS_SCATT_F*N_SPINS*N_SPINS]; + Float2 accum[N_GAMMAS_SCATT_I*N_GAMMAS_SCATT_F*N_SPINS*N_SPINS]; for(int i = 0 ; i -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -16,7 +16,7 @@ __global__ void T2_kernel( KernelArr listGammas_i, KernelArr accum[N_GAMMAS_SCATT_I*N_GAMMAS_SCATT_F*N_SPINS*N_SPINS]; + Float2 accum[N_GAMMAS_SCATT_I*N_GAMMAS_SCATT_F*N_SPINS*N_SPINS]; for(int i = 0 ; i +#include "PLEGMA_kernel_utils.cuh" #include -#include -#include -#include -#include -#include -#include -#include +#include "PLEGMA_scattreductions.cuh" +#include "PLEGMA_scattreductionsV2.cuh" +#include "PLEGMA_scattreductionsV3.cuh" +#include "PLEGMA_scattreductionsV4.cuh" +#include "PLEGMA_scattreductionsV5.cuh" +#include "PLEGMA_scattreductionsV6.cuh" +#include "PLEGMA_scattreductionsV6_red.cuh" using namespace plegma; diff --git a/lib/kernels/PLEGMA_scattreductionsV2.cuh b/lib/kernels/PLEGMA_scattreductionsV2.cuh index 82b9d8c7..eacb45cb 100644 --- a/lib/kernels/PLEGMA_scattreductionsV2.cuh +++ b/lib/kernels/PLEGMA_scattreductionsV2.cuh @@ -1,5 +1,5 @@ -#include -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -14,7 +14,7 @@ __global__ void V2_kernel( vectorTex vectorPhi, KernelArr int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_SPINS*N_SPINS*N_COLS]; + Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_SPINS*N_SPINS*N_COLS]; for(int i = 0 ; i -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -14,7 +14,7 @@ __global__ void V3_kernel( vectorTex vectorPhi, KernelArr int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_COLS]; + Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_COLS]; for(int i = 0 ; i -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -14,7 +14,7 @@ __global__ void V4_kernel( vectorTex vectorPhi, KernelArr int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_SPINS*N_SPINS*N_COLS]; + Float2 accum[N_GAMMAS_SCATT*N_SPINS*N_SPINS*N_SPINS*N_COLS]; for(int i = 0 ; i -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -13,7 +13,7 @@ __global__ void V5_kernel( vectorTex vectorPhi1, vectorTex vecto int tid = blockIdx.x/grid3D; int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_SPINS*N_SPINS*N_COLS]; + Float2 accum[N_SPINS*N_SPINS*N_COLS]; for(int i = 0 ; i -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -14,7 +14,7 @@ __global__ void V6_kernel( vectorTex vectorPhi1, vectorTex vecto int t=it+tid; if(t>=maxT) t=(source.w%DGC_localL[DIM_T])+t-maxT; int vid = sid3D + t*DGC_localVolume3D; - register Float2 accum[N_SPINS*N_SPINS*N_SPINS*N_SPINS*N_COLS]; + Float2 accum[N_SPINS*N_SPINS*N_SPINS*N_SPINS*N_COLS]; for(int i = 0 ; i < N_SPINS*N_SPINS*N_SPINS*N_SPINS*N_COLS ; i++){ accum[i] = 0.; } diff --git a/lib/kernels/PLEGMA_scattreductionsV6_red.cuh b/lib/kernels/PLEGMA_scattreductionsV6_red.cuh index 165a0cd7..e3885127 100644 --- a/lib/kernels/PLEGMA_scattreductionsV6_red.cuh +++ b/lib/kernels/PLEGMA_scattreductionsV6_red.cuh @@ -1,5 +1,5 @@ -#include -#include <../../include/PLEGMA_gammas.h> +#include "PLEGMA_kernel_utils.cuh" +#include "../../include/PLEGMA_gammas.h" using namespace plegma; @@ -17,7 +17,7 @@ __global__ void V6_RED_kernel( vectorTex vectorPhi1, vectorTex v const unsigned short N_S1C=N_SPINS*N_COLS; const unsigned short N_S2C=N_SPINS*N_S1C; - register Float2 accum[N_GAMMAS_SCATT*N_S2C]; + Float2 accum[N_GAMMAS_SCATT*N_S2C]; for(int i = 0 ; i < N_GAMMAS_SCATT*N_S2C ; i++){ accum[i] = 0.; } diff --git a/lib/kernels/PLEGMA_seqSourceNucleon.cu b/lib/kernels/PLEGMA_seqSourceNucleon.cu index a47e7ec4..e7be573f 100644 --- a/lib/kernels/PLEGMA_seqSourceNucleon.cu +++ b/lib/kernels/PLEGMA_seqSourceNucleon.cu @@ -1,7 +1,7 @@ -#include -#include +#include +#include #include -#include +#include #ifdef PLEGMA_NUCLEON_3PF_FIX_SINK static const __device__ short int NtoN_indices[16][4] = {0,1,0,1,0,1,1,0,0,1,2,3,0,1,3,2,1,0,0,1,1,0,1,0,1,0,2,3,1,0,3,2,2,3,0,1,2,3,1,0,2,3,2,3,2,3,3,2,3,2,0,1,3,2,1,0,3,2,2,3,3,2,3,2}; diff --git a/lib/kernels/PLEGMA_shifts.cuh b/lib/kernels/PLEGMA_shifts.cuh index 1b98401f..90b71b2b 100644 --- a/lib/kernels/PLEGMA_shifts.cuh +++ b/lib/kernels/PLEGMA_shifts.cuh @@ -1,4 +1,4 @@ -#include +#include "PLEGMA_kernel_utils.cuh" using namespace plegma; template @@ -6,8 +6,8 @@ static __global__ void shifts1_kernel(pFloat2 in, pFloat2 out, int int sid = blockIdx.x*blockDim.x + threadIdx.x; if(sid >= in.volume()) return; in.setSid(sid); - if(dirOr<4) in.shift(dirOr%4); //:bring the elem from behind the current pos - else in.shift(dirOr%4); + if(dirOr<4) in.template shift(dirOr%4); //:bring the elem from behind the current pos + else in.template shift(dirOr%4); out.setSid(sid); for(int i = 0 ; i < in.site_size; i++){ out.set(i, in.get(i)); @@ -29,10 +29,10 @@ static __global__ void shifts2_kernel(pFloat2 in, pFloat2 out, int int sid = blockIdx.x*blockDim.x + threadIdx.x; if(sid >= in.volume()) return; in.setSid(sid); - if(dirOr1<4 && dirOr2<4) in.shift(dirOr1%4,dirOr2%4); - else if(dirOr1<4) in.shift(dirOr1%4,dirOr2%4); - else if(dirOr2<4) in.shift(dirOr1%4,dirOr2%4); - else in.shift(dirOr1%4,dirOr2%4); + if(dirOr1<4 && dirOr2<4) in.template shift(dirOr1%4,dirOr2%4); + else if(dirOr1<4) in.template shift(dirOr1%4,dirOr2%4); + else if(dirOr2<4) in.template shift(dirOr1%4,dirOr2%4); + else in.template shift(dirOr1%4,dirOr2%4); out.setSid(sid); for(int i = 0 ; i < in.site_size; i++){ out.set(i, in.get(i)); @@ -54,14 +54,14 @@ static __global__ void shifts3_kernel(pFloat2 in, pFloat2 out, int int sid = blockIdx.x*blockDim.x + threadIdx.x; if(sid >= in.volume()) return; in.setSid(sid); - if(dirOr1<4 && dirOr2<4 && dirOr3<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr1<4 && dirOr2<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr2<4 && dirOr3<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr1<4 && dirOr3<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr1<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr2<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else if(dirOr3<4) in.shift(dirOr1%4,dirOr2%4,dirOr3%4); - else in.shift(dirOr1%4,dirOr2%4,dirOr3%4); + if(dirOr1<4 && dirOr2<4 && dirOr3<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr1<4 && dirOr2<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr2<4 && dirOr3<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr1<4 && dirOr3<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr1<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr2<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else if(dirOr3<4) in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); + else in.template shift(dirOr1%4,dirOr2%4,dirOr3%4); out.setSid(sid); for(int i = 0 ; i < in.site_size; i++){ out.set(i, in.get(i)); diff --git a/lib/kernels/PLEGMA_su3field.cuh b/lib/kernels/PLEGMA_su3field.cuh index 0a47f6be..ff9d3a53 100644 --- a/lib/kernels/PLEGMA_su3field.cuh +++ b/lib/kernels/PLEGMA_su3field.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" #include #include using namespace plegma; diff --git a/lib/kernels/PLEGMA_threep_local.cu b/lib/kernels/PLEGMA_threep_local.cu index f26f01d0..3aa2b4ea 100644 --- a/lib/kernels/PLEGMA_threep_local.cu +++ b/lib/kernels/PLEGMA_threep_local.cu @@ -1,8 +1,8 @@ #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include using namespace plegma; @@ -102,8 +102,8 @@ static void threep_local_host(ProfileStruct &ps, Float2 *result, KernelArr listGammas; listGammas.size = gammas.size(); - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + listGammas.array=(GAMMAS *)device_malloc( gammas.size()*sizeof(GAMMAS)); + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), qudaMemcpyHostToDevice); if(HGC_verbosity > 2) if(corr.hasSource()) @@ -113,14 +113,14 @@ static void threep_local_host(ProfileStruct &ps, Float2 *result, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture>(prop1); auto propTex2 = toTexture>(prop2); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int mu=-1, nu=-1, c1=-1, c2=-1; @@ -136,10 +136,10 @@ static void threep_local_host(ProfileStruct &ps, Float2 *result, threep_local_device <<>> (d_partial_block, *propTex1, *propTex2, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -161,8 +161,8 @@ static void threep_local_host(ProfileStruct &ps, Float2 *result, exit: hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); + device_free(d_partial_block); + device_free(listGammas.array); } template diff --git a/lib/kernels/PLEGMA_threep_noe.cu b/lib/kernels/PLEGMA_threep_noe.cu index b3b3f91a..e9b8cbcc 100644 --- a/lib/kernels/PLEGMA_threep_noe.cu +++ b/lib/kernels/PLEGMA_threep_noe.cu @@ -1,9 +1,10 @@ #include #include -#include -#include -#include -#include +#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" template __global__ void threep_noe_device(Float2* block2, @@ -33,22 +34,22 @@ __global__ void threep_noe_device(Float2* block2, #pragma unroll for(int dir = 0; dir < N_DIMS; dir++) { // term x, x, x+dir - prop1Tex.get(prop1,vid); gaugeTex.get(su3,dir,vid); prop2Tex.get(prop2,vid,dir); + prop1Tex.get(prop1,vid); gaugeTex.get(su3,dir,vid); prop2Tex.template get(prop2,vid,dir); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3); noeV[dir] += trace_gamma_S(gamma_dir[dir],NOROT,R) - trace_gamma_S(ONE,NOROT,R); //term x, x-dir, x-dir - gaugeTex.get(su3,dir,vid,dir); prop2Tex.get(prop2,vid,dir); + gaugeTex.template get(su3,dir,vid,dir); prop2Tex.template get(prop2,vid,dir); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3); noeV[dir] += trace_gamma_S(gamma_dir[dir],NOROT,R) + trace_gamma_S(ONE,NOROT,R); //term x+dir, x, x - prop1Tex.get(prop1,vid,dir); gaugeTex.get(su3,dir,vid); prop2Tex.get(prop2,vid); + prop1Tex.template get(prop1,vid,dir); gaugeTex.get(su3,dir,vid); prop2Tex.get(prop2,vid); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3); noeV[dir] += trace_gamma_S(gamma_dir[dir],NOROT,R) + trace_gamma_S(ONE,NOROT,R); //term x-dir, x-dir, x - prop1Tex.get(prop1,vid,dir); gaugeTex.get(su3,dir,vid,dir); + prop1Tex.template get(prop1,vid,dir); gaugeTex.template get(su3,dir,vid,dir); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3); noeV[dir] += trace_gamma_S(gamma_dir[dir],NOROT,R) - trace_gamma_S(ONE,NOROT,R); } @@ -94,15 +95,15 @@ static void threep_noe_host(ProfileStruct &ps, Float2 *result, PLEGMA_Co Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); auto propTex2 = toTexture(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { int t_step = std::min(t_size-it, time_step); dim3 grid = ps.tp.grid; @@ -110,10 +111,10 @@ static void threep_noe_host(ProfileStruct &ps, Float2 *result, PLEGMA_Co threep_noe_device <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, it, t_step, maxT, source, signProps, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block , d_partial_block , (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -134,7 +135,7 @@ static void threep_noe_host(ProfileStruct &ps, Float2 *result, PLEGMA_Co exit: hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); + device_free(d_partial_block); } template diff --git a/lib/kernels/PLEGMA_threep_oneD.cu b/lib/kernels/PLEGMA_threep_oneD.cu index be4f9b04..55ddda87 100644 --- a/lib/kernels/PLEGMA_threep_oneD.cu +++ b/lib/kernels/PLEGMA_threep_oneD.cu @@ -1,10 +1,11 @@ #include #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" +#include using namespace plegma; template @@ -52,19 +53,19 @@ __global__ void threep_oneD_device(Float2* block2, // This has been generated by python/generate_derivative.py // The order has been optmized such that 3 data accesses have been commented out // + term x, x, x+dir - texture1.get(prop1,vid); gaugeTex.get(su3,dir,vid); texture2.get(prop2,vid,dir); + texture1.get(prop1,vid); gaugeTex.get(su3,dir,vid); texture2.template get(prop2,vid,dir); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3,mu,nu,c1,c2); // - term x, x-dir^, x-dir - /*texture1.get(prop1,vid);*/ gaugeTex.get(su3,dir,vid,dir); texture2.get(prop2,vid,dir); + /*texture1.get(prop1,vid);*/ gaugeTex.template get(su3,dir,vid,dir); texture2.template get(prop2,vid,dir); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3,mu,nu,c1,c2); // + term x-dir, x-dir, x - texture1.get(prop1,vid,dir); /*gaugeTex.get(su3,dir,vid,dir);*/ texture2.get(prop2,vid); + texture1.template get(prop1,vid,dir); /*gaugeTex.get(su3,dir,vid,dir);*/ texture2.get(prop2,vid); partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3,mu,nu,c1,c2); // - term x+dir, x^, x - texture1.get(prop1,vid,dir); gaugeTex.get(su3,dir,vid); /*texture2.get(prop2,vid);*/ + texture1.template get(prop1,vid,dir); gaugeTex.get(su3,dir,vid); /*texture2.get(prop2,vid);*/ partial_trace_mul_Prop_G_Prop(R,prop1,prop2,su3,mu,nu,c1,c2); // END REGION @@ -124,8 +125,8 @@ static void threep_oneD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C KernelArr listGammas; listGammas.size = gammas.size(); - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + listGammas.array=(GAMMAS*)device_malloc(gammas.size()*sizeof(GAMMAS)); + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), qudaMemcpyHostToDevice); if(HGC_verbosity > 2) if(corr.hasSource()) @@ -135,15 +136,15 @@ static void threep_oneD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture>(prop1); auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int mu=-1, nu=-1, c1=-1, c2=-1; @@ -160,10 +161,10 @@ static void threep_oneD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); + //error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -185,8 +186,8 @@ static void threep_oneD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C exit: hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); + device_free(d_partial_block); + device_free(listGammas.array); } template diff --git a/lib/kernels/PLEGMA_threep_qgq.cu b/lib/kernels/PLEGMA_threep_qgq.cu index 2915f0e3..1c51aa1f 100644 --- a/lib/kernels/PLEGMA_threep_qgq.cu +++ b/lib/kernels/PLEGMA_threep_qgq.cu @@ -1,10 +1,10 @@ #include #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include #include @@ -95,7 +95,7 @@ static void threep_qgq_host(ProfileStruct &ps, Float2 *result, listGammas.size = gammas.size(); //cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); listGammas.array=(GAMMAS*)device_malloc(gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), qudaMemcpyHostToDevice); if(HGC_verbosity > 2) if(corr.hasSource()) @@ -114,8 +114,8 @@ static void threep_qgq_host(ProfileStruct &ps, Float2 *result, su3_2 RFmunu((Float2*) Fmunu.D_elem()+shift, su3_l.Field_length(), Fmunu.is4D(), false); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { int t_step = std::min(t_size-it, time_step); dim3 grid = ps.tp.grid; @@ -126,10 +126,10 @@ static void threep_qgq_host(ProfileStruct &ps, Float2 *result, toField2(su3_l), RFmunu, toField2(su3_r), listGammas, it, t_step, maxT, source, signProps, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -154,8 +154,8 @@ static void threep_qgq_host(ProfileStruct &ps, Float2 *result, exit: hostFree(h_partial_block, alloc_size*sizeof(Float)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); + device_free(d_partial_block); + device_free(listGammas.array); } diff --git a/lib/kernels/PLEGMA_threep_staple.cu b/lib/kernels/PLEGMA_threep_staple.cu index 923f27ef..698c01dd 100644 --- a/lib/kernels/PLEGMA_threep_staple.cu +++ b/lib/kernels/PLEGMA_threep_staple.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include using namespace plegma; template @@ -104,8 +104,8 @@ static void threep_staple_host(ProfileStruct &ps, Float2 *result, auto propTex2 = toTexture(prop2); auto su3tex = toTexture(su3); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int mu=-1, nu=-1, c1=-1, c2=-1; @@ -122,10 +122,10 @@ static void threep_staple_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *su3tex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -147,12 +147,12 @@ static void threep_staple_host(ProfileStruct &ps, Float2 *result, exit: hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); + device_free(d_partial_block); + device_free(listGammas.array); } template -static void threep_staple(PLEGMA_Correlator &corr, +void threep_staple(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& su3, std::vector& gammas,bool isZfac){ #ifdef PLEGMA_NUCLEON_3PF_FIX_SINK diff --git a/lib/kernels/PLEGMA_threep_threeD_part1.cu b/lib/kernels/PLEGMA_threep_threeD_part1.cu index 46f96d4d..a7692284 100644 --- a/lib/kernels/PLEGMA_threep_threeD_part1.cu +++ b/lib/kernels/PLEGMA_threep_threeD_part1.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include using namespace plegma; @@ -169,7 +169,8 @@ static void threep_threeD_part1_host(ProfileStruct &ps, Float2 *result, listGammas.size = gammas.size(); listGammas.array=(GAMMAS*)device_malloc(gammas.size()*sizeof(GAMMAS)); // cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + listGammas.array=(GAMMAS*)device_malloc(gammas.size()*sizeof(GAMMAS)); + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), qudaMemcpyHostToDevice); if(HGC_verbosity > 2) if(corr.hasSource()) @@ -187,8 +188,8 @@ static void threep_threeD_part1_host(ProfileStruct &ps, Float2 *result, auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int dir1 = (et/(N_DIMS-1)/(N_DIMS-2)) % N_DIMS; @@ -220,10 +221,10 @@ static void threep_threeD_part1_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, dir1,dir2,dir3,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_threep_threeD_part2.cu b/lib/kernels/PLEGMA_threep_threeD_part2.cu index 8a59cd31..5ff28170 100644 --- a/lib/kernels/PLEGMA_threep_threeD_part2.cu +++ b/lib/kernels/PLEGMA_threep_threeD_part2.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include #include @@ -186,8 +186,8 @@ static void threep_threeD_part2_host(ProfileStruct &ps, Float2 *result, auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int dir1 = (et/(N_DIMS-1)/(N_DIMS-2)) % N_DIMS; @@ -219,10 +219,10 @@ static void threep_threeD_part2_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, dir1,dir2,dir3,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_threep_threeD_part3.cu b/lib/kernels/PLEGMA_threep_threeD_part3.cu index c0aa8327..5f31e940 100644 --- a/lib/kernels/PLEGMA_threep_threeD_part3.cu +++ b/lib/kernels/PLEGMA_threep_threeD_part3.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include #include @@ -186,8 +186,8 @@ static void threep_threeD_part3_host(ProfileStruct &ps, Float2 *result, auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; + //cudaError_t error=cudaPeekAtLastError(); + //if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int dir1 = (et/(N_DIMS-1)/(N_DIMS-2)) % N_DIMS; @@ -219,10 +219,10 @@ static void threep_threeD_part3_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, dir1,dir2,dir3,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_threep_threeD_part4.cu b/lib/kernels/PLEGMA_threep_threeD_part4.cu index a8f5cbd7..cacef16c 100644 --- a/lib/kernels/PLEGMA_threep_threeD_part4.cu +++ b/lib/kernels/PLEGMA_threep_threeD_part4.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include #include @@ -188,8 +188,8 @@ static void threep_threeD_part4_host(ProfileStruct &ps, Float2 *result, auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int dir1 = (et/(N_DIMS-1)/(N_DIMS-2)) % N_DIMS; @@ -221,10 +221,10 @@ static void threep_threeD_part4_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, dir1,dir2,dir3,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_threep_twoD.cu b/lib/kernels/PLEGMA_threep_twoD.cu index 3fde0846..521f5c7a 100644 --- a/lib/kernels/PLEGMA_threep_twoD.cu +++ b/lib/kernels/PLEGMA_threep_twoD.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" #include #include #include @@ -50,67 +50,67 @@ __global__ void threep_twoD_device(Float2* block2, // This has been generated by python/generate_derivative.py // The order has been optmized such that 18 data accesses have been commented out // + term x, x, x+dir1, x+dir1+dir2 - texture1.get(prop1,vid); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid,dir1); texture2.get(prop2,vid,dir1,dir2); + texture1.get(prop1,vid); gaugeTex.get(su3_1,dir1,vid); gaugeTex.template get(su3_2,dir2,vid,dir1); texture2.template get(prop2,vid,dir1,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x, x, x+dir1-dir2^, x+dir1-dir2 - /*texture1.get(prop1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); texture2.get(prop2,vid,dir1,dir2); + /*texture1.get(prop1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.template get(su3_2,dir2,vid,dir1,dir2); texture2.template get(prop2,vid,dir1,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x+dir2, x+dir2, x+dir1^, x+dir1 - texture1.get(prop1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); texture2.get(prop2,vid,dir1); + texture1.template get(prop1,vid,dir2); gaugeTex.template get(su3_1,dir1,vid,dir2); gaugeTex.template get(su3_2,dir2,vid,dir1); texture2.template get(prop2,vid,dir1); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x-dir2, x-dir2, x+dir1-dir2, x+dir1 - texture1.get(prop1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*texture2.get(prop2,vid,dir1);*/ + texture1.template get(prop1,vid,dir2); gaugeTex.template get(su3_1,dir1,vid,dir2); gaugeTex.template get(su3_2,dir2,vid,dir1,dir2); /*texture2.get(prop2,vid,dir1);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x, x-dir1^, x-dir1, x-dir1+dir2 - texture1.get(prop1,vid); gaugeTex.get(su3_1,dir1,vid,dir1); gaugeTex.get(su3_2,dir2,vid,dir1); texture2.get(prop2,vid,dir1,dir2); + texture1.get(prop1,vid); gaugeTex.template get(su3_1,dir1,vid,dir1); gaugeTex.template get(su3_2,dir2,vid,dir1); texture2.template get(prop2,vid,dir1,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x, x-dir1^, x-dir1-dir2^, x-dir1-dir2 - /*texture1.get(prop1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid,dir1,dir2); texture2.get(prop2,vid,dir1,dir2); + /*texture1.get(prop1,vid);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.template get(su3_2,dir2,vid,dir1,dir2); texture2.template get(prop2,vid,dir1,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x+dir2, x-dir1+dir2^, x-dir1^, x-dir1 - texture1.get(prop1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1); texture2.get(prop2,vid,dir1); + texture1.template get(prop1,vid,dir2); gaugeTex.template get(su3_1,dir1,vid,dir1,dir2); gaugeTex.template get(su3_2,dir2,vid,dir1); texture2.template get(prop2,vid,dir1); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x-dir2, x-dir1-dir2^, x-dir1-dir2, x-dir1 - texture1.get(prop1,vid,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); gaugeTex.get(su3_2,dir2,vid,dir1,dir2); /*texture2.get(prop2,vid,dir1);*/ + texture1.template get(prop1,vid,dir2); gaugeTex.template get(su3_1,dir1,vid,dir1,dir2); gaugeTex.template get(su3_2,dir2,vid,dir1,dir2); /*texture2.get(prop2,vid,dir1);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x+dir1, x^, x, x+dir2 - texture1.get(prop1,vid,dir1); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid); texture2.get(prop2,vid,dir2); + texture1.template get(prop1,vid,dir1); gaugeTex.get(su3_1,dir1,vid); gaugeTex.get(su3_2,dir2,vid); texture2.template get(prop2,vid,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x+dir1, x^, x-dir2^, x-dir2 - /*texture1.get(prop1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.get(su3_2,dir2,vid,dir2); texture2.get(prop2,vid,dir2); + /*texture1.get(prop1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid);*/ gaugeTex.template get(su3_2,dir2,vid,dir2); texture2.template get(prop2,vid,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x-dir1, x-dir1, x-dir2^, x-dir2 - texture1.get(prop1,vid,dir1); gaugeTex.get(su3_1,dir1,vid,dir1); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*texture2.get(prop2,vid,dir2);*/ + texture1.template get(prop1,vid,dir1); gaugeTex.template get(su3_1,dir1,vid,dir1); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*texture2.get(prop2,vid,dir2);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x-dir1, x-dir1, x, x+dir2 - /*texture1.get(prop1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid); texture2.get(prop2,vid,dir2); + /*texture1.get(prop1,vid,dir1);*/ /*gaugeTex.get(su3_1,dir1,vid,dir1);*/ gaugeTex.get(su3_2,dir2,vid); texture2.template get(prop2,vid,dir2); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x+dir1+dir2, x+dir2^, x^, x - texture1.get(prop1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ texture2.get(prop2,vid); + texture1.template get(prop1,vid,dir1,dir2); gaugeTex.template get(su3_1,dir1,vid,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ texture2.get(prop2,vid); partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x-dir1+dir2, x-dir1+dir2, x^, x - texture1.get(prop1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ /*texture2.get(prop2,vid);*/ + texture1.template get(prop1,vid,dir1,dir2); gaugeTex.template get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid);*/ /*texture2.get(prop2,vid);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // - term x+dir1-dir2, x-dir2^, x-dir2, x - texture1.get(prop1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir2); gaugeTex.get(su3_2,dir2,vid,dir2); /*texture2.get(prop2,vid);*/ + texture1.template get(prop1,vid,dir1,dir2); gaugeTex.template get(su3_1,dir1,vid,dir2); gaugeTex.template get(su3_2,dir2,vid,dir2); /*texture2.get(prop2,vid);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // + term x-dir1-dir2, x-dir1-dir2, x-dir2, x - texture1.get(prop1,vid,dir1,dir2); gaugeTex.get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*texture2.get(prop2,vid);*/ + texture1.template get(prop1,vid,dir1,dir2); gaugeTex.template get(su3_1,dir1,vid,dir1,dir2); /*gaugeTex.get(su3_2,dir2,vid,dir2);*/ /*texture2.get(prop2,vid);*/ partial_trace_mul_Prop_G1_G2_Prop(R,prop1,prop2,su3_1,su3_2,mu,nu,c1,c2); // END REGION @@ -186,8 +186,8 @@ static void threep_twoD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C auto propTex2 = toTexture>(prop2); auto gaugetex = toTexture(gauge); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { for(int et=0; et < extra; et++) { int dir1 = (et/(N_DIMS-1)) % N_DIMS; @@ -208,10 +208,10 @@ static void threep_twoD_host(ProfileStruct &ps, Float2 *result, PLEGMA_C <<>> (d_partial_block, *propTex1, *propTex2, *gaugetex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms, dir1,dir2,mu,nu,c1,c2); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2), qudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; diff --git a/lib/kernels/PLEGMA_threep_wilsonLine.cu b/lib/kernels/PLEGMA_threep_wilsonLine.cu index ddbbd5d7..f54848dd 100644 --- a/lib/kernels/PLEGMA_threep_wilsonLine.cu +++ b/lib/kernels/PLEGMA_threep_wilsonLine.cu @@ -1,9 +1,9 @@ #include #include -#include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_getSet.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_threep.cuh" using namespace plegma; template @@ -75,8 +75,8 @@ static void threep_wilsonLine_host(ProfileStruct &ps, Float2 *result, KernelArr listGammas; listGammas.size = gammas.size(); - cudaMalloc((void**)&listGammas.array, gammas.size()*sizeof(GAMMAS)); - cudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), cudaMemcpyHostToDevice); + listGammas.array=(GAMMAS*)device_malloc( gammas.size()*sizeof(GAMMAS)); + qudaMemcpy(listGammas.array, gammas.data(), gammas.size()*sizeof(GAMMAS), qudaMemcpyHostToDevice); if(HGC_verbosity > 2) if(corr.hasSource()) @@ -86,15 +86,15 @@ static void threep_wilsonLine_host(ProfileStruct &ps, Float2 *result, Float2 *h_partial_block = NULL; Float2 *d_partial_block = NULL; - cudaMalloc((void**)&d_partial_block, alloc_size * sizeof(Float2) ); + d_partial_block=(Float2*)device_malloc(alloc_size * sizeof(Float2) ); hostMalloc(h_partial_block, alloc_size*sizeof(Float2)); auto propTex1 = toTexture(prop1); auto propTex2 = toTexture(prop2); auto su3tex = toTexture(su3); - cudaError_t error=cudaPeekAtLastError(); - if(error != cudaSuccess || h_partial_block==NULL) goto exit; +// cudaError_t error=cudaPeekAtLastError(); +// if(error != cudaSuccess || h_partial_block==NULL) goto exit; for(int it=0; it < t_size; it+=time_step) { int t_step = std::min(t_size-it, time_step); dim3 grid = ps.tp.grid; @@ -103,10 +103,10 @@ static void threep_wilsonLine_host(ProfileStruct &ps, Float2 *result, <<>> (d_partial_block, *propTex1, *propTex2, *su3tex, listGammas, it, t_step, maxT, source, signProps, runFT, *moms); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; - cudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , cudaMemcpyDeviceToHost); - error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; + qudaMemcpy(h_partial_block, d_partial_block, (alloc_size/time_step)*t_step*sizeof(Float2) , qudaMemcpyDeviceToHost); +// error=cudaPeekAtLastError(); if(error != cudaSuccess) goto exit; if(runFT==true){ int accumX = ps.tp.grid.x/time_step; @@ -127,12 +127,12 @@ static void threep_wilsonLine_host(ProfileStruct &ps, Float2 *result, exit: hostFree(h_partial_block, alloc_size*sizeof(FloatC)); - cudaFree(d_partial_block); - cudaFree(listGammas.array); + device_free(d_partial_block); + device_free(listGammas.array); } template -static void threep_wilsonLine(PLEGMA_Correlator &corr, +void threep_wilsonLine(PLEGMA_Correlator &corr, PLEGMA_Propagator& prop1, PLEGMA_Propagator& prop2, int signProps, PLEGMA_Su3field& su3, std::vector& gammas){ #ifdef PLEGMA_NUCLEON_3PF_FIX_SINK diff --git a/lib/kernels/PLEGMA_topocharge.cuh b/lib/kernels/PLEGMA_topocharge.cuh index fe064ddb..b1a74eeb 100644 --- a/lib/kernels/PLEGMA_topocharge.cuh +++ b/lib/kernels/PLEGMA_topocharge.cuh @@ -1,5 +1,5 @@ #pragma once -#include +#include "PLEGMA_kernel_utils.cuh" #include using namespace plegma; @@ -19,11 +19,11 @@ __device__ void clover_term( Float2 C[N_COLS][N_COLS], gaugeTex // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] gaugeTex.get(G1,dir1,sid); - gaugeTex.get(G2,dir2,sid,dir1); + gaugeTex.template get(G2,dir2,sid,dir1); mul_G_G(G3,G1,G2); - gaugeTex.get(G1,dir1,sid,dir2); + gaugeTex.template get(G1,dir1,sid,dir2); gaugeTex.get(G2,dir2,sid); mul_Gdag_Gdag(G4,G1,G2); @@ -33,12 +33,12 @@ __device__ void clover_term( Float2 C[N_COLS][N_COLS], gaugeTex // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] gaugeTex.get(G1,dir2,sid); - gaugeTex.get(G2,dir1,sid,dir1,dir2); + gaugeTex.template get(G2,dir1,sid,dir1,dir2); mul_G_Gdag(G3,G1,G2); - gaugeTex.get(G1,dir2,sid,dir1); - gaugeTex.get(G2,dir1,sid,dir1); + gaugeTex.template get(G1,dir2,sid,dir1); + gaugeTex.template get(G2,dir1,sid,dir1); mul_Gdag_G(G4,G1,G2); @@ -46,13 +46,13 @@ __device__ void clover_term( Float2 C[N_COLS][N_COLS], gaugeTex G_plus_aG( C, P, 1.); // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] - gaugeTex.get(G1,dir1,sid,dir1); - gaugeTex.get(G2,dir2,sid,dir1,dir2); + gaugeTex.template get(G1,dir1,sid,dir1); + gaugeTex.template get(G2,dir2,sid,dir1,dir2); mul_Gdag_Gdag(G3,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 - gaugeTex.get(G1,dir1,sid,dir1,dir2); - gaugeTex.get(G2,dir2,sid,dir2); + gaugeTex.template get(G1,dir1,sid,dir1,dir2); + gaugeTex.template get(G2,dir2,sid,dir2); mul_G_G(G4,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 @@ -60,12 +60,12 @@ __device__ void clover_term( Float2 C[N_COLS][N_COLS], gaugeTex G_plus_aG( C, P, 1.); // term trace[U^{i}(id) * U^{j}(id+i) * U^{i+}(id+j) * U^{j+}(id)] - gaugeTex.get(G1,dir2,sid,dir2); - gaugeTex.get(G2,dir1,sid,dir2); + gaugeTex.template get(G1,dir2,sid,dir2); + gaugeTex.template get(G2,dir1,sid,dir2); mul_Gdag_G(G3,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 - gaugeTex.get(G1,dir2,sid,dir1,dir2); + gaugeTex.template get(G1,dir2,sid,dir1,dir2); gaugeTex.get(G2,dir1,sid); mul_G_Gdag(G4,G1,G2); // flops = N_COLS*N_COLS*N_COLS*2 @@ -124,12 +124,12 @@ __device__ void plaquette( Float2 C[N_COLS][N_COLS], gaugeTex &g //G3 = U^{dir1}(id) * U^{dir2}(id+dir1) gaugeTex.get(G1,dir1,sid); - gaugeTex.get(G2,dir2,sid,dir1); + gaugeTex.template get(G2,dir2,sid,dir1); mul_G_G(G3,G1,G2); //G4 = U^{dir1+}(id+dir2) * U^{dir2+}(id) - gaugeTex.get(G1,dir1,sid,dir2); + gaugeTex.template get(G1,dir1,sid,dir2); gaugeTex.get(G2,dir2,sid); mul_Gdag_Gdag(G4,G1,G2); @@ -185,7 +185,7 @@ static Float calcTopoCharge(gaugeTex gaugeTex, TOPO_CHARGE_DEF charge_de Float *d_partial_Q = NULL; h_partial_Q = (Float*) malloc(gridDim.x * sizeof(Float) ); if(h_partial_Q == NULL) errorQuda("Error allocate memory for host partial plaq"); - cudaMalloc((void**)&d_partial_Q, gridDim.x * sizeof(Float)); + d_partial_Q=(Float*)device_malloc(gridDim.x * sizeof(Float)); switch(charge_def){ case PLAQUETTE: @@ -197,8 +197,8 @@ static Float calcTopoCharge(gaugeTex gaugeTex, TOPO_CHARGE_DEF charge_de } checkQudaError(); - cudaMemcpy(h_partial_Q, d_partial_Q , gridDim.x * sizeof(Float) , cudaMemcpyDeviceToHost); - cudaFree(d_partial_Q); + qudaMemcpy(h_partial_Q, d_partial_Q , gridDim.x * sizeof(Float) , qudaMemcpyDeviceToHost); + device_free(d_partial_Q); checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++){ @@ -253,12 +253,12 @@ static Float calcPlaqClovDef(gaugeTex gaugeTex){ Float *d_partial_Plaq = NULL; h_partial_Plaq = (Float*) malloc(gridDim.x * sizeof(Float) ); if(h_partial_Plaq == NULL) errorQuda("Error allocate memory for host partial plaq"); - cudaMalloc((void**)&d_partial_Plaq, gridDim.x * sizeof(Float)); + d_partial_Plaq=(Float*)device_malloc(gridDim.x * sizeof(Float)); calcPlaqClovDef_kernel<<>>( gaugeTex, d_partial_Plaq ); - cudaMemcpy(h_partial_Plaq, d_partial_Plaq , gridDim.x * sizeof(Float) , cudaMemcpyDeviceToHost); - cudaFree(d_partial_Plaq); + qudaMemcpy(h_partial_Plaq, d_partial_Plaq , gridDim.x * sizeof(Float) , qudaMemcpyDeviceToHost); + device_free(d_partial_Plaq); checkQudaError(); for(int i = 0 ; i < gridDim.x ; i++) diff --git a/lib/kernels/PLEGMA_u1gauge_utils.cuh b/lib/kernels/PLEGMA_u1gauge_utils.cuh index d857492f..bbe1e128 100644 --- a/lib/kernels/PLEGMA_u1gauge_utils.cuh +++ b/lib/kernels/PLEGMA_u1gauge_utils.cuh @@ -1,5 +1,5 @@ -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_kernel_tuner.cuh" using namespace plegma; using namespace quda; diff --git a/lib/kernels/PLEGMA_vector_utils.cuh b/lib/kernels/PLEGMA_vector_utils.cuh index 1bd35f0b..fe4d090f 100644 --- a/lib/kernels/PLEGMA_vector_utils.cuh +++ b/lib/kernels/PLEGMA_vector_utils.cuh @@ -1,8 +1,9 @@ +#pragma once #include #include -#include -#include -#include +#include "PLEGMA_kernel_utils.cuh" +#include "PLEGMA_gammas.cuh" +#include "PLEGMA_kernel_tuner.cuh" #include using namespace plegma; using namespace quda; @@ -268,8 +269,9 @@ static void compute_rms(const PLEGMA_Vector3D &vec, std::vector &lis d_absPsi=(Float *)device_malloc(absPsi.size() * sizeof(Float)); checkQudaError(); - cudaMemcpy(d_listR2,listR2.data(), listR2.size() * sizeof(int), cudaMemcpyHostToDevice); checkQudaError(); - cudaMemset(d_absPsi,0,absPsi.size() * sizeof(Float)); checkQudaError(); + qudaMemcpy(d_listR2,listR2.data(), listR2.size() * sizeof(int), qudaMemcpyHostToDevice); checkQudaError(); + qudaMemset(d_absPsi,0,absPsi.size() * sizeof(Float)); + //checkQudaError(); thrust::counting_iterator first(0); thrust::counting_iterator last = first + HGC_localVolume3D; typedef thrust::device_ptr > DpF2; @@ -279,9 +281,9 @@ static void compute_rms(const PLEGMA_Vector3D &vec, std::vector &lis zipTplIntDev2 z1 = thrust::make_zip_iterator(thrust::make_tuple(first,y)); zipTplIntDev2 z2 = thrust::make_zip_iterator(thrust::make_tuple(last,y+HGC_localVolume3D)); thrust::for_each(z1,z2,computeRMS(sourceposition[0],sourceposition[1],sourceposition[2],listR2.size(),d_listR2,d_absPsi)); - cudaMemcpy(absPsi.data(), d_absPsi, absPsi.size() * sizeof(Float), cudaMemcpyDeviceToHost); checkQudaError(); - cudaFree(d_listR2); - cudaFree(d_absPsi); + qudaMemcpy(absPsi.data(), d_absPsi, absPsi.size() * sizeof(Float), qudaMemcpyDeviceToHost); checkQudaError(); + device_free(d_listR2); + device_free(d_absPsi); } template diff --git a/lib/targets/cuda/target_cuda.cmake b/lib/targets/cuda/target_cuda.cmake new file mode 100644 index 00000000..54d5db78 --- /dev/null +++ b/lib/targets/cuda/target_cuda.cmake @@ -0,0 +1,133 @@ +# ###################################################################################################################### +# malloc.cpp uses both the driver and runtime api +# So we need to find the CUDA_cuda_LIBRARY (driver api) or the stub version +find_library(CUDA_cuda_LIBRARY cuda HINTS ${CUDA_TOOLKIT_ROOT_DIR}/lib/ ${CUDA_TOOLKIT_ROOT_DIR}/lib/stubs) +#target_link_libraries(plegma PUBLIC ${CUDA_cuda_LIBRARY}) +# CUDA specific part of CMakeLists +include(CheckLanguage) +check_language(CUDA) + +set(PLEGMA_TARGET_CUDA ON) + +# using dirs in LD_LIBRARY_PATh for searching for libraries +string(REPLACE ":" ";" LIBRARY_DIRS $ENV{LD_LIBRARY_PATH}) + +set(DEFAULT_GPU_ARCH sm_70) +set(GPU_ARCH ${DEFAULT_GPU_ARCH} CACHE STRING "set the GPU architecture (sm_20, sm_21, sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_70)") +set_property(CACHE GPU_ARCH PROPERTY STRINGS sm_20 sm_21 sm_30 sm_35 sm_37 sm_50 sm_52 sm_60 sm_70) + +# Use CUDA textures +set(PLEGMA_TEXTURE TRUE CACHE BOOL "Wheater to use or not CUDA textures") +mark_as_advanced(PLEGMA_TEXTURE) +if(PLEGMA_TEXTURE) + add_definitions(-DPLEGMA_TEXTURE) +else() + +endif() + +####################################################################### +# everything below here is processing the setup +####################################################################### +# we need to check for some packages +find_package(PythonInterp) + +# solve compiler issues with CUDA and tuples +if( (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.5) AND (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER 9.0) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)) + message(FATAL_ERROR "This library will have compilation problems with CUDA 9.1 and gcc 6") +endif() +if( (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER 9.1) AND (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.3) ) + message(FATAL_ERROR "This library will have compilation problems with CUDA 9.2 and gcc < 6") +endif() + +LIST(APPEND CUDA_LIBS ${CUDA_cublas_LIBRARY} ) +LIST(APPEND CUDA_LIBS ${CUDA_cufft_LIBRARY} ) +LIST(APPEND CUDA_LIBS ${CUDA_curand_LIBRARY} ) +LIST(APPEND CUDA_LIBS ${CUDA_nvrtc_LIBRARY} ${CUDA_nvToolsExt_LIBRARY}) + +find_package(LibDL) +LIST(APPEND CUDA_LIBS ${LIBDL_LIBRARIES}) + +add_definitions(-DMULTI_GPU) + +include_directories(SYSTEM ${CUDA_INCLUDE_DIRS}) +include_directories(kernels) + + +# GPU ARCH +STRING(REGEX REPLACE sm_ "" COMP_CAP ${GPU_ARCH}) +SET(COMP_CAP "${COMP_CAP}0") +add_definitions(-D__COMPUTE_CAPABILITY__=${COMP_CAP}) + + +# NVCC FLAGS independet off build type +if(NOT USING_CUDA_LANG_SUPPORT) + set(CUDA_NVCC_FLAGS "-std c++17 -arch=${GPU_ARCH} -ftz=true -prec-div=false -prec-sqrt=false") +else() + set(CUDA_NVCC_FLAGS "-ftz=true -prec-div=false -prec-sqrt=false") + set(CMAKE_CUDA_FLAGS "-arch=${GPU_ARCH}" CACHE STRING "Flags used by the CUDA compiler" FORCE) +endif() + +# some clang warnings shouds be warning even when turning warnings into errors +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CLANG_NOERROR "-Wno-error=unused-private-field") +# this is a hack to get colored diagnostics back when using Ninja and clang + if(CMAKE_GENERATOR MATCHES "Ninja") + set(CLANG_FORCE_COLOR "-fcolor-diagnostics") + endif() +endif() + +## define CUDA flags when CMake < 3.8 +set(CUDA_NVCC_FLAGS_DEVEL ${CUDA_NVCC_FLAGS} -Wno-deprecated-gpu-targets -Xcompiler -Wno-unknown-pragmas,-Wno-unused-function,-Wno-unused-local-typedef,-Wno-unused-private-field -O3 -lineinfo CACHE STRING + "Flags used by the CUDA compiler during regular development builds." + FORCE ) +set(CUDA_NVCC_FLAGS_STRICT ${CUDA_NVCC_FLAGS_DEVEL} CACHE STRING + "Flags used by the CUDA compiler during strict jenkins builds." + FORCE ) +set(CUDA_NVCC_FLAGS_RELEASE ${CUDA_NVCC_FLAGS} -O3 -w CACHE STRING + "Flags used by the C++ compiler during release builds." + FORCE ) +set(CUDA_NVCC_FLAGS_HOSTDEBUG ${CUDA_NVCC_FLAGS} -g -lineinfo -DHOST_DEBUG CACHE STRING + "Flags used by the C++ compiler during host-debug builds." + FORCE ) +set(CUDA_NVCC_FLAGS_DEVICEDEBUG ${CUDA_NVCC_FLAGS} -G CACHE STRING + "Flags used by the C++ compiler during device-debug builds." + FORCE ) +set(CUDA_NVCC_FLAGS_DEBUG ${CUDA_NVCC_FLAGS} -g -DHOST_DEBUG -G CACHE STRING + "Flags used by the C++ compiler during full (host+device) debug builds." + FORCE ) + +## define CUDA flags when CMake >= 3.8 +set(CMAKE_CUDA_FLAGS_DEVEL "${CUDA_NVCC_FLAGS} -Wno-deprecated-gpu-targets -Xcompiler -Wno-unknown-pragmas,-Wno-unused-function,-Wno-unused-local-typedef,-Wno-unused-private-field -O3 -lineinfo" CACHE STRING + "Flags used by the CUDA compiler during regular development builds." + FORCE ) +set(CMAKE_CUDA_FLAGS_STRICT "${CMAKE_CUDA_FLAGS_DEVEL}" CACHE STRING + "Flags used by the CUDA compiler during strict jenkins builds." + FORCE ) +set(CMAKE_CUDA_FLAGS_RELEASE "${CUDA_NVCC_FLAGS} -O3 -w" CACHE STRING + "Flags used by the CUDA compiler during release builds." + FORCE ) +set(CMAKE_CUDA_FLAGS_HOSTDEBUG "${CUDA_NVCC_FLAGS} -g -lineinfo -DHOST_DEBUG" CACHE STRING + "Flags used by the C++ compiler during host-debug builds." + FORCE ) +set(CMAKE_CUDA_FLAGS_DEVICEDEBUG "${CUDA_NVCC_FLAGS} -G" CACHE STRING + "Flags used by the C++ compiler during device-debug builds." + FORCE ) +set(CMAKE_CUDA_FLAGS_DEBUG "${CUDA_NVCC_FLAGS} -g -DHOST_DEBUG -G" CACHE STRING + "Flags used by the C++ compiler during full (host+device) debug builds." + FORCE ) + + +mark_as_advanced(CUDA_NVCC_FLAGS_DEVEL) +mark_as_advanced(CUDA_NVCC_FLAGS_STRICT) +mark_as_advanced(CUDA_NVCC_FLAGS_HOSTDEBUG) +mark_as_advanced(CUDA_NVCC_FLAGS_DEVICEDEBUG) +mark_as_advanced(CMAKE_CUDA_FLAGS_DEVEL) +mark_as_advanced(CMAKE_CUDA_FLAGS_STRICT) +mark_as_advanced(CMAKE_CUDA_FLAGS_HOSTDEBUG) +mark_as_advanced(CMAKE_CUDA_FLAGS_DEVICEDEBUG) + +# make one library + +target_include_directories(plegma PUBLIC $ + $) +target_include_directories(plegma PUBLIC ${QUDA_HOME}/include/targets/cuda) diff --git a/lib/targets/hip/CMakeLists.txt b/lib/targets/hip/CMakeLists.txt new file mode 100644 index 00000000..26a3351c --- /dev/null +++ b/lib/targets/hip/CMakeLists.txt @@ -0,0 +1,4 @@ +# ######################################################################################################################### +# Additional sources +target_sources(plegma_cpp PRIVATE quda_api.cpp device.cpp malloc.cpp blas_lapack_hipblas.cpp comm_target.cpp) + diff --git a/lib/targets/hip/target_hip.cmake b/lib/targets/hip/target_hip.cmake new file mode 100644 index 00000000..4e8c18a6 --- /dev/null +++ b/lib/targets/hip/target_hip.cmake @@ -0,0 +1,137 @@ +# ###################################################################################################################### +# HIP specific part of CMakeLists +include(CheckLanguage) +check_language(HIP) + +set(plegma_TARGET_HIP ON) + +if(DEFINED ENV{GPU_ARCH}) + set(DEFAULT_GPU_ARCH $ENV{GPU_ARCH}) +else() + set(DEFAULT_GPU_ARCH gfx908) +endif() + +set(GPU_ARCH + ${DEFAULT_GPU_ARCH} + CACHE STRING "set the GPU architecture (gfx906 gfx908 gfx90a)") +set_property(CACHE GPU_ARCH PROPERTY STRINGS gfx906 gfx908 gfx90a) + +set(CMAKE_HIP_ARCHITECTURES "${GPU_ARCH}") +set(GPU_TARGETS "${GPU_ARCH}") + +mark_as_advanced(GPU_TARGETS) +mark_as_advanced(CMAKE_HIP_ARCHITECTURES) +message(STATUS "Building for GPU Architectures: ${GPU_ARCH}") + +find_package(HIP) +find_package(hipfft REQUIRED) +find_package(hiprand REQUIRED) +find_package(rocrand REQUIRED) +find_package(hipblas REQUIRED) +find_package(rocblas REQUIRED) +find_package(hipcub REQUIRED) +find_package(rocprim REQUIRED) + + +# ###################################################################################################################### +# define CUDA flags +set(CMAKE_HIP_HOST_COMPILER + "${CMAKE_CXX_COMPILER}" + CACHE FILEPATH "Host compiler to be used by hip") +set(CMAKE_HIP_STANDARD ${QUDA_CXX_STANDARD}) +set(CMAKE_HIP_STANDARD_REQUIRED True) +mark_as_advanced(CMAKE_HIP_HOST_COMPILER) + +set(CMAKE_HIP_FLAGS_DEVEL + "-g -O3 " + CACHE STRING "Flags used by the CUDA compiler during regular development builds.") +set(CMAKE_HIP_FLAGS_STRICT + "-g -O3" + CACHE STRING "Flags used by the CUDA compiler during strict jenkins builds.") +set(CMAKE_HIP_FLAGS_RELEASE + "-O3 -w" + CACHE STRING "Flags used by the CUDA compiler during release builds.") +set(CMAKE_HIP_FLAGS_HOSTDEBUG + "-g" + CACHE STRING "Flags used by the C++ compiler during host-debug builds.") +set(CMAKE_HIP_FLAGS_DEBUG + "-g -G" + CACHE STRING "Flags used by the C++ compiler during full (host+device) debug builds.") +set(CMAKE_HIP_FLAGS_SANITIZE + "-g " + CACHE STRING "Flags used by the C++ compiler during sanitizer debug builds.") + +mark_as_advanced(CMAKE_HIP_FLAGS_DEVEL) +mark_as_advanced(CMAKE_HIP_FLAGS_STRICT) +mark_as_advanced(CMAKE_HIP_FLAGS_RELEASE) +mark_as_advanced(CMAKE_HIP_FLAGS_DEBUG) +mark_as_advanced(CMAKE_HIP_FLAGS_HOSTDEBUG) +mark_as_advanced(CMAKE_HIP_FLAGS_SANITIZE) +enable_language(HIP) +message(STATUS "HIP Compiler is" ${CMAKE_HIP_COMPILER}) +message(STATUS "Compiler ID is " ${CMAKE_HIP_COMPILER_ID}) + +# ###################################################################################################################### +# CUDA specific QUDA options options +set(QUDA_HETEROGENEOUS_ATOMIC OFF) +mark_as_advanced(QUDA_HETEROGENEOUS_ATOMIC) + +# ###################################################################################################################### +# CUDA specific variables +set_target_properties(plegma PROPERTIES HIP_ARCHITECTURES ${CMAKE_HIP_ARCHITECTURES}) + +# QUDA_HASH for tunecache +set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${GPU_ARCH},hip_version=${CMAKE_HIP_COMPILER_VERSION}) +set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-${GPU_ARCH}") + + + +# ###################################################################################################################### +# cuda specific compile options + +# Use CUDA textures +set(PLEGMA_TEXTURE TRUE CACHE BOOL "Wheater to use or not CUDA textures") +mark_as_advanced(PLEGMA_TEXTURE) +if(PLEGMA_TEXTURE) + add_definitions(-DPLEGMA_TEXTURE) +else() + +endif() + + +#target_include_directories(plegma PRIVATE ${CMAKE_SOURCE_DIR}/include/targets/hip) +#target_include_directories(plegma PUBLIC $ +# $) + + +#set_source_files_properties(block_orthogonalize.cu PROPERTIES COMPILE_OPTIONS "-mllvm;-pragma-unroll-threshold=4096") + +add_definitions(-DMULTI_GPU) + +target_compile_options( + plegma + PRIVATE -Wall + -Wextra + -Wno-unknown-pragmas + -Wno-unused-result + -Wno-deprecated-register -dc + -fgpu-rdc #--amdgpu-target=gfx90a + -fopenmp + $<$:-Werror + -Wno-error=pass-failed> + $<$:-fsanitize=address + -fsanitize=undefined>) + + set_source_files_properties( ${PLEGMA_CU_OBJS} PROPERTIES LANGUAGE HIP) +# malloc.cpp uses both the driver and runtime api So we need to find the CUDA_CUDA_LIBRARY (driver api) or the stub +# version for cmake 3.8 and later this has been integrated into FindCUDALibs.cmake +target_link_libraries(plegma PUBLIC hip::hiprand roc::rocrand hip::hipcub roc::rocprim_hip) +target_link_libraries(plegma PUBLIC roc::hipblas roc::rocblas) + +target_include_directories(plegma PUBLIC ${ROCM_PATH}/hipfft/include) +target_include_directories(plegma PUBLIC ${QUDA_HOME}/include/targets/hip) +target_link_libraries(plegma PUBLIC hip::hipfft) + +#add_subdirectory(targets/hip) + +install(FILES ${CMAKE_SOURCE_DIR}/cmake/find_target_hip_dependencies.cmake DESTINATION lib/cmake/PLEGMA) diff --git a/lib/version.cpp b/lib/version.cpp new file mode 100644 index 00000000..1c8114ba --- /dev/null +++ b/lib/version.cpp @@ -0,0 +1,5 @@ +#ifdef GITVERSION +const char* gitversion = GITVERSION ; +#else +const char* gitversion; +#endif diff --git a/plegma/CMakeLists.txt b/plegma/CMakeLists.txt index c2a9c252..c610d6e0 100644 --- a/plegma/CMakeLists.txt +++ b/plegma/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PLEGMA_LIBS PLEGMA PLEGMAutils ${CMAKE_THREAD_LIBS_INIT} ${QUDA_LIB} ${CUDA_LIBS} cuda) +set(PLEGMA_LIBS plegma PLEGMAutils ${CMAKE_THREAD_LIBS_INIT} ${QUDA_LIB} ${CUDA_LIBS} cublas cuda) LIST(APPEND PLEGMA_LIBS ${MPI_CXX_LIBRARIES}) @@ -37,168 +37,165 @@ endif(PLEGMA_MKL) cuda_add_executable(Plaquette Plaquette.cpp) target_link_libraries(Plaquette ${PLEGMA_LIBS}) -cuda_add_executable(Topo Topo.cpp) +add_executable(Topo Topo.cpp) target_link_libraries(Topo ${PLEGMA_LIBS}) -cuda_add_executable(Calc_2pt Calc_2pt.cpp) +add_executable(Calc_2pt Calc_2pt.cpp) target_link_libraries(Calc_2pt ${PLEGMA_LIBS}) -cuda_add_executable(Calc_2pt_wilson Calc_2pt_wilson.cpp) +add_executable(Calc_2pt_wilson Calc_2pt_wilson.cpp) target_link_libraries(Calc_2pt_wilson ${PLEGMA_LIBS}) -cuda_add_executable(Tetraquark Tetraquark.cpp) +add_executable(Tetraquark Tetraquark.cpp) target_link_libraries(Tetraquark ${PLEGMA_LIBS}) -cuda_add_executable(TetraquarkBCUD TetraquarkBCUD.cpp) +add_executable(TetraquarkBCUD TetraquarkBCUD.cpp) target_link_libraries(TetraquarkBCUD ${PLEGMA_LIBS}) -cuda_add_executable(TestFrame EXCLUDE_FROM_ALL TestFrame.cpp) +add_executable(TestFrame EXCLUDE_FROM_ALL TestFrame.cpp) target_link_libraries(TestFrame ${PLEGMA_LIBS}) -cuda_add_executable(StochasticTetraquark StochasticTetraquark.cpp) +add_executable(StochasticTetraquark StochasticTetraquark.cpp) target_link_libraries(StochasticTetraquark ${PLEGMA_LIBS}) -cuda_add_executable(StochasticTetraquarkBCUD StochasticTetraquarkBCUD.cpp) +add_executable(StochasticTetraquarkBCUD StochasticTetraquarkBCUD.cpp) target_link_libraries(StochasticTetraquarkBCUD ${PLEGMA_LIBS}) -cuda_add_executable(ComputeLightStochProp ComputeLightStochProp.cpp) +add_executable(ComputeLightStochProp ComputeLightStochProp.cpp) target_link_libraries(ComputeLightStochProp ${PLEGMA_LIBS}) -cuda_add_executable(ComputeLightPointProp ComputeLightPointProp.cpp) +add_executable(ComputeLightPointProp ComputeLightPointProp.cpp) target_link_libraries(ComputeLightPointProp ${PLEGMA_LIBS}) -cuda_add_executable(Calc_2pt_charmonium Calc_2pt_charmonium.cpp) +add_executable(Calc_2pt_charmonium Calc_2pt_charmonium.cpp) target_link_libraries(Calc_2pt_charmonium ${PLEGMA_LIBS}) -cuda_add_executable(PDFs_nucleon PDFs_nucleon.cpp) +add_executable(PDFs_nucleon PDFs_nucleon.cpp) target_link_libraries(PDFs_nucleon ${PLEGMA_LIBS}) -cuda_add_executable(TMDPDFs_nucleon TMDPDFs_nucleon.cpp) +add_executable(TMDPDFs_nucleon TMDPDFs_nucleon.cpp) target_link_libraries(TMDPDFs_nucleon ${PLEGMA_LIBS}) -cuda_add_executable(PDFs_qgq_nucleon PDFs_qgq_nucleon.cpp) +add_executable(PDFs_qgq_nucleon PDFs_qgq_nucleon.cpp) target_link_libraries(PDFs_qgq_nucleon ${PLEGMA_LIBS}) -cuda_add_executable(nucleon_2pt_3pt nucleon_2pt_3pt.cpp) +add_executable(nucleon_2pt_3pt nucleon_2pt_3pt.cpp) target_link_libraries(nucleon_2pt_3pt ${PLEGMA_LIBS}) -cuda_add_executable(nucleon_2pt_3pt_npi nucleon_2pt_3pt_npi.cpp) +add_executable(nucleon_2pt_3pt_npi nucleon_2pt_3pt_npi.cpp) target_link_libraries(nucleon_2pt_3pt_npi ${PLEGMA_LIBS}) -cuda_add_executable(mesons_3pt mesons_3pt.cpp) +add_executable(mesons_3pt mesons_3pt.cpp) target_link_libraries(mesons_3pt ${PLEGMA_LIBS}) -cuda_add_executable(correcting_ZM EXCLUDE_FROM_ALL correcting_ZM.cpp) +add_executable(correcting_ZM EXCLUDE_FROM_ALL correcting_ZM.cpp) target_link_libraries(correcting_ZM ${PLEGMA_LIBS}) -cuda_add_executable(Source EXCLUDE_FROM_ALL Source.cpp) +add_executable(Source EXCLUDE_FROM_ALL Source.cpp) target_link_libraries(Source ${PLEGMA_LIBS}) -cuda_add_executable(Random EXCLUDE_FROM_ALL Random.cpp) +add_executable(Random EXCLUDE_FROM_ALL Random.cpp) target_link_libraries(Random ${PLEGMA_LIBS}) -cuda_add_executable(Dilution EXCLUDE_FROM_ALL Dilution.cpp) +add_executable(Dilution EXCLUDE_FROM_ALL Dilution.cpp) target_link_libraries(Dilution ${PLEGMA_LIBS}) -cuda_add_executable(testLinksSmearings EXCLUDE_FROM_ALL testLinksSmearings.cpp) +add_executable(testLinksSmearings EXCLUDE_FROM_ALL testLinksSmearings.cpp) target_link_libraries(testLinksSmearings ${PLEGMA_LIBS}) -cuda_add_executable(piNdiagrams_3pt_N_Npi EXCLUDE_FROM_ALL piNdiagrams_3pt_N_Npi.cpp) -target_link_libraries(piNdiagrams_3pt_N_Npi ${PLEGMA_LIBS}) - - -cuda_add_executable(testHDF5 EXCLUDE_FROM_ALL testHDF5.cpp) +add_executable(testHDF5 EXCLUDE_FROM_ALL testHDF5.cpp) target_link_libraries(testHDF5 ${PLEGMA_LIBS}) -cuda_add_executable(qLoops qLoops.cpp) +add_executable(qLoops qLoops.cpp) target_link_libraries(qLoops ${PLEGMA_LIBS}) -cuda_add_executable(qLoops_WilsonLine qLoops_WilsonLine.cpp) +add_executable(qLoops_WilsonLine qLoops_WilsonLine.cpp) target_link_libraries(qLoops_WilsonLine ${PLEGMA_LIBS}) -cuda_add_executable(gLoops_WilsonLine gLoops_WilsonLine.cpp) +add_executable(gLoops_WilsonLine gLoops_WilsonLine.cpp) target_link_libraries(gLoops_WilsonLine ${PLEGMA_LIBS}) -cuda_add_executable(FT EXCLUDE_FROM_ALL FT.cpp) +add_executable(FT EXCLUDE_FROM_ALL FT.cpp) target_link_libraries(FT ${PLEGMA_LIBS}) -cuda_add_executable(LowModesCalc LowModesCalc.cpp) +add_executable(LowModesCalc LowModesCalc.cpp) target_link_libraries(LowModesCalc ${PLEGMA_LIBS}) -cuda_add_executable(MG_tuner MG_tuner.cpp) +add_executable(MG_tuner MG_tuner.cpp) target_link_libraries(MG_tuner ${PLEGMA_LIBS}) -cuda_add_executable(benchmark benchmark.cpp) +add_executable(benchmark benchmark.cpp) target_link_libraries(benchmark ${PLEGMA_LIBS}) -cuda_add_executable(TMDWFs_mesons TMDWFs_mesons.cpp) +add_executable(TMDWFs_mesons TMDWFs_mesons.cpp) target_link_libraries(TMDWFs_mesons ${PLEGMA_LIBS}) -cuda_add_executable(quark_rms EXCLUDE_FROM_ALL quark_rms.cpp) +add_executable(quark_rms EXCLUDE_FROM_ALL quark_rms.cpp) target_link_libraries(quark_rms ${PLEGMA_LIBS}) -cuda_add_executable(Zfac_Vgg EXCLUDE_FROM_ALL Zfac_Vgg.cpp) +add_executable(Zfac_Vgg EXCLUDE_FROM_ALL Zfac_Vgg.cpp) target_link_libraries(Zfac_Vgg ${PLEGMA_LIBS}) -cuda_add_executable(testShifts EXCLUDE_FROM_ALL testShifts.cpp) +add_executable(testShifts EXCLUDE_FROM_ALL testShifts.cpp) target_link_libraries(testShifts ${PLEGMA_LIBS}) -cuda_add_executable(Zfac Zfac.cpp) +add_executable(Zfac Zfac.cpp) target_link_libraries(Zfac ${PLEGMA_LIBS}) -cuda_add_executable(ZfacStaple ZfacStaple.cpp) +add_executable(ZfacStaple ZfacStaple.cpp) target_link_libraries(ZfacStaple ${PLEGMA_LIBS}) -cuda_add_executable(SoftFunction SoftFunction.cpp) +add_executable(SoftFunction SoftFunction.cpp) target_link_libraries(SoftFunction ${PLEGMA_LIBS}) -cuda_add_executable(SoftFunctionWall SoftFunctionWall.cpp) +add_executable(SoftFunctionWall SoftFunctionWall.cpp) target_link_libraries(SoftFunctionWall ${PLEGMA_LIBS}) -cuda_add_executable(WallSourceNucleon WallSourceNucleon.cpp) +add_executable(WallSourceNucleon WallSourceNucleon.cpp) target_link_libraries(WallSourceNucleon ${PLEGMA_LIBS}) -cuda_add_executable(TMDPDFs_pion TMDPDFs_pion.cpp) +add_executable(TMDPDFs_pion TMDPDFs_pion.cpp) target_link_libraries(TMDPDFs_pion ${PLEGMA_LIBS}) if (PLEGMA_SCATTERING_CONTRACTIONS) - cuda_add_executable(testScattReductions EXCLUDE_FROM_ALL testScattReductions.cpp) + add_executable(testScattReductions EXCLUDE_FROM_ALL testScattReductions.cpp) target_link_libraries(testScattReductions ${PLEGMA_LIBS}) - cuda_add_executable(piNdiagrams piNdiagrams.cpp) + add_executable(piNdiagrams piNdiagrams.cpp) target_link_libraries(piNdiagrams ${PLEGMA_LIBS}) - cuda_add_executable(nucleon_delta_test EXCLUDE_FROM_ALL nucleon_delta_test.cpp) + add_executable(nucleon_delta_test EXCLUDE_FROM_ALL nucleon_delta_test.cpp) target_link_libraries(nucleon_delta_test ${PLEGMA_LIBS}) - cuda_add_executable(nucleon_3pt_n_npi EXCLUDE_FROM_ALL nucleon_3pt_n_npi.cpp) + add_executable(piNdiagrams_oet EXCLUDE_FROM_ALL piNdiagrams_oet.cpp) + target_link_libraries(piNdiagrams_oet ${PLEGMA_LIBS}) + add_executable(nucleon_3pt_n_npi EXCLUDE_FROM_ALL nucleon_3pt_n_npi.cpp) target_link_libraries(nucleon_3pt_n_npi ${PLEGMA_LIBS}) - cuda_add_executable(piNdiagrams_oet piNdiagrams_oet.cpp) target_link_libraries(piNdiagrams_oet ${PLEGMA_LIBS}) - cuda_add_executable(checking_Doet checking_Doet.cpp) + add_executable(checking_Doet checking_Doet.cpp) target_link_libraries(checking_Doet ${PLEGMA_LIBS}) - cuda_add_executable(omegamass EXCLUDE_FROM_ALL omegamass.cpp) + add_executable(omegamass EXCLUDE_FROM_ALL omegamass.cpp) target_link_libraries(omegamass ${PLEGMA_LIBS}) endif(PLEGMA_SCATTERING_CONTRACTIONS) -cuda_add_executable(gaugeFixing gaugeFixing.cpp) +add_executable(gaugeFixing gaugeFixing.cpp) target_link_libraries(gaugeFixing ${PLEGMA_LIBS}) -cuda_add_executable(invert invert.cpp) +add_executable(invert invert.cpp) target_link_libraries(invert ${PLEGMA_LIBS}) -cuda_add_executable(polarN EXCLUDE_FROM_ALL polarN.cpp) +add_executable(polarN EXCLUDE_FROM_ALL polarN.cpp) target_link_libraries(polarN ${PLEGMA_LIBS}) -cuda_add_executable(Zfac_GI Zfac_GI.cpp) +add_executable(Zfac_GI Zfac_GI.cpp) target_link_libraries(Zfac_GI ${PLEGMA_LIBS}) -cuda_add_executable(Zfac_GI_meson3pt Zfac_GI_meson3pt.cpp) +add_executable(Zfac_GI_meson3pt Zfac_GI_meson3pt.cpp) target_link_libraries(Zfac_GI_meson3pt ${PLEGMA_LIBS}) -cuda_add_executable(mesons_2pt mesons_2pt.cpp) +add_executable(mesons_2pt mesons_2pt.cpp) target_link_libraries(mesons_2pt ${PLEGMA_LIBS}) -cuda_add_executable(mesons_simple mesons_simple.cpp) +add_executable(mesons_simple mesons_simple.cpp) target_link_libraries(mesons_simple ${PLEGMA_LIBS}) -cuda_add_executable(Zfac_diagGI Zfac_diagGI.cpp) +add_executable(Zfac_diagGI Zfac_diagGI.cpp) target_link_libraries(Zfac_diagGI ${PLEGMA_LIBS}) diff --git a/plegma/Plaquette.cpp b/plegma/Plaquette.cpp index 91f98519..fd61ec19 100644 --- a/plegma/Plaquette.cpp +++ b/plegma/Plaquette.cpp @@ -1,6 +1,10 @@ #include #include +#ifdef __NVCC__ #include +#elif defined (__HIP__) +#include +#endif using namespace plegma; using namespace quda; @@ -15,7 +19,11 @@ int main(int argc, char **argv) // Allocation done on BOTH, DEVICE and HOST { +#ifdef __NVCC__ cudaProfilerStart(); +#elif defined (__HIP__) + hipProfilerStart(); +#endif PLEGMA_Gauge gauge(BOTH); // Reading from Lime file and loading to device gauge.readFile(latfile, LIME_FORMAT); @@ -30,7 +38,11 @@ int main(int argc, char **argv) // Loading to QUDA and computing plaquette also there initGaugeQuda(gauge, false, QUDA_SU3_LINKS); plaqQuda(); +#ifdef __NVCC__ cudaProfilerStop(); +#elif defined (__HIP__) + hipProfilerStop(); +#endif } finalize(); diff --git a/plegma/benchmark.cpp b/plegma/benchmark.cpp index 4c7d8ade..6a0906e7 100644 --- a/plegma/benchmark.cpp +++ b/plegma/benchmark.cpp @@ -3,8 +3,16 @@ #include #include #include +#if defined (__NVCC__) #include +#else +//#include +#endif +#ifdef __NVCC__ #include "nvToolsExt.h" +#else +#include "roctx.h" +#endif using namespace plegma; using namespace quda; @@ -12,7 +20,7 @@ using namespace quda; int n_benchmark = 100; double t_max = 10; template -void PLEGMA_benchmark(T *obj, out (T1::*function)(types1...),std::string name, types&&... kArgs){ +void PLEGMA_benchmark(T *obj, out (T1::*function)(types1...),std::string name, types... kArgs){ std::vector timing; @@ -22,11 +30,27 @@ void PLEGMA_benchmark(T *obj, out (T1::*function)(types1...),std::string name, t double t0 = MPI_Wtime(); for(int i=0; i*function)(kArgs...); +#ifdef __NVCC__ nvtxRangePop(); +#else + roctxRangePop(); +#endif +#ifdef __NVCC__ cudaProfilerStop(); +#else + hipProfilerStop(); +#endif timing.push_back(MPI_Wtime()-t1); // Setting an hard break after t_max sec if(i > 1 && MPI_Wtime()-t0 > t_max) break; @@ -82,15 +106,19 @@ int main(int argc, char **argv) { PLEGMA_Vector vector_a, vector_b; PLEGMA_Gauge3D gauge3D; PLEGMA_Vector3D vector3D_a, vector3D_b; - - // Benchmark of the APEsmearing function - PLEGMA_benchmark(&gauge_b,&PLEGMA_Gauge::APEsmearing, "APEsmearing (1 iter)", gauge_a, 1, 0.5, 3); +// +//void PLEGMA_benchmark(T *obj, out (T1::*function)(types1...),std::string name, types&&... kArgs){ + // Benchmark of the APEsmearing function + // void APEsmearing(PLEGMA_Gauge &uin, int nSmear, double alpha, int D3D4); + + PLEGMA_benchmark(&gauge_b,static_cast::*)(PLEGMA_Gauge &, int, double , int )> + (&PLEGMA_Gauge::APEsmearing), "APEsmearing (1 iter)", &gauge_a, 1, 0.5, 3); // Benchmark gaussian smearing - PLEGMA_benchmark(&vector3D_a,&PLEGMA_Vector::gaussianSmearing,"Gaussian Smearing 3D on one timeslice (1 iter)",vector3D_b, gauge3D, 1, 0.2); + //PLEGMA_benchmark(&vector3D_a,&PLEGMA_Vector::gaussianSmearing,"Gaussian Smearing 3D on one timeslice (1 iter)",vector3D_b, gauge3D, 1, 0.2); // Benchmark gaussian smearing - PLEGMA_benchmark(&vector_a,&PLEGMA_Vector::gaussianSmearing,"Gaussian Smearing on all timeslice (1 iter)",vector_b, gauge_a, 1, 0.2); + //PLEGMA_benchmark(&vector_a,&PLEGMA_Vector::gaussianSmearing,"Gaussian Smearing on all timeslice (1 iter)",vector_b, gauge_a, 1, 0.2); } if(run({"twop"})) { @@ -100,14 +128,14 @@ int main(int argc, char **argv) { PLEGMA_Correlator corr(corr_space, source, maxQsq); // Benchmark Meson contration - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractMesons,"Contraction mesons",prop_a, prop_b); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractMesons,"Contraction mesons",prop_a, prop_b); // Benchmark Baryons contractions - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractBaryons,"Contraction Baryons",prop_a, prop_b); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractBaryons,"Contraction Baryons",prop_a, prop_b); PLEGMA_Propagator prop_c, prop_d; // Benchmark Baryons contractions - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractBaryonsUDSC,"Contraction Baryons Proj",prop_a, prop_b, prop_c, prop_d, false, false); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractBaryonsUDSC,"Contraction Baryons Proj",prop_a, prop_b, prop_c, prop_d, false, false); } #ifdef PLEGMA_NUCLEON_3PF_FIX_SINK @@ -119,8 +147,8 @@ int main(int argc, char **argv) { void (PLEGMA_Vector3D::*seqSourceNucleon)(PLEGMA_Propagator3D &, PLEGMA_Propagator3D &, WHICHPROJECTOR, WHICHPARTICLE, int, int) = &PLEGMA_Vector3D::seqSourceNucleon; // Benchmark Sequential source for(int i=0; i<(int) N_PROJS; i++) { - PLEGMA_benchmark(&vector, seqSourceNucleon, "Sequential source proton P=" + std::to_string(i), prop_a, prop_b, (WHICHPROJECTOR) i, PROTON, 0, 0); - PLEGMA_benchmark(&vector, seqSourceNucleon, "Sequential source neutron P=" + std::to_string(i), prop_a, prop_b, (WHICHPROJECTOR) i, NEUTRON, 0, 0); + //PLEGMA_benchmark(&vector, seqSourceNucleon, "Sequential source proton P=" + std::to_string(i), prop_a, prop_b, (WHICHPROJECTOR) i, PROTON, 0, 0); + //PLEGMA_benchmark(&vector, seqSourceNucleon, "Sequential source neutron P=" + std::to_string(i), prop_a, prop_b, (WHICHPROJECTOR) i, NEUTRON, 0, 0); } } @@ -134,10 +162,10 @@ int main(int argc, char **argv) { // Benchmark three point functions - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_local,"Contraction local",prop_a, prop_b, +1, gammas,false); - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_oneD,"Contraction one derivative",prop_a, prop_b, gauge, +1, gammas, false); - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_twoD,"Contraction third derivative",prop_a, prop_b, gauge, +1, gammas, false); - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_noe,"Contraction Noether",prop_a, prop_b, gauge, +1); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_local,"Contraction local",prop_a, prop_b, +1, gammas,false); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_oneD,"Contraction one derivative",prop_a, prop_b, gauge, +1, gammas, false); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_twoD,"Contraction third derivative",prop_a, prop_b, gauge, +1, gammas, false); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_noe,"Contraction Noether",prop_a, prop_b, gauge, +1); } if(run({"PDFs"})) { @@ -151,19 +179,19 @@ int main(int argc, char **argv) { momSmScale[0] = {0.7071, 0.7071}; // Benchmark of the momentum smearing - PLEGMA_benchmark(&gauge,&PLEGMA_Gauge::scaleDirWise,"Momentum smearing (scale 1 dir)",momSmScale); + //PLEGMA_benchmark(&gauge,&PLEGMA_Gauge::scaleDirWise,"Momentum smearing (scale 1 dir)",momSmScale); // Benchmark contraction 3pt - PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_wilsonLine,"Contraction 3pt",prop_a, prop_b, su3, +1, gammas, 0, (std::string)""); + //PLEGMA_benchmark(&corr,&PLEGMA_Correlator::contractNucleonThrp_wilsonLine,"Contraction 3pt",prop_a, prop_b, su3, +1, gammas, 0, (std::string)""); // Benchmark Wilson line update - PLEGMA_benchmark(&su3,&PLEGMA_Su3field::wilsonLineUpdate,"Update of the Wilson line",su3_a, su3_b,4+2, false); + //PLEGMA_benchmark(&su3,&PLEGMA_Su3field::wilsonLineUpdate,"Update of the Wilson line",su3_a, su3_b,4+2, false); // Benchmark shift routine - PLEGMA_benchmark(&prop_b,static_cast::*)(PLEGMA_Field &, short )>(&PLEGMA_Field::shift),"Shift routine", prop_a, 2); + //PLEGMA_benchmark(&prop_b,static_cast::*)(PLEGMA_Field &, short )>(&PLEGMA_Field::shift),"Shift routine", prop_a, 2); // Benchmark stout smearing - PLEGMA_benchmark(&gauge,&PLEGMA_Gauge::stoutSmearing,"Stout smearing (1 step)",gauge, 1,0.4,3); + // PLEGMA_benchmark(&gauge,&PLEGMA_Gauge::stoutSmearing,"Stout smearing (1 step)",gauge, 1,0.4,3); } #endif @@ -174,12 +202,12 @@ int main(int argc, char **argv) { // Benchmark standard one-end trick // Since the function is overloaded we need to select one version of it - PLEGMA_benchmark(&loops,static_cast::*)(PLEGMA_Vector &, PLEGMA_Vector &, double , bool )> - (&PLEGMA_QLoops::oneEnd_trick),"Loops one-end trick",vector_a,vector_b,-1.,true); +// PLEGMA_benchmark(&loops,static_cast::*)(PLEGMA_Vector &, PLEGMA_Vector &, double , bool )> + // (&PLEGMA_QLoops::oneEnd_trick),"Loops one-end trick",vector_a,vector_b,-1.,true); // Benchmark standard one-end trick - PLEGMA_benchmark(&ft,&PLEGMA_FT::apply,"Loops FT",loops,FT_GEMV,-1); +// PLEGMA_benchmark(&ft,&PLEGMA_FT::apply,"Loops FT",loops,FT_GEMV,-1); } finalize(); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 51d4e37b..7d71e446 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -3,6 +3,13 @@ file(GLOB PLEGMA_UTILS "*.cpp" ) -cuda_add_library(PLEGMAutils STATIC ${PLEGMA_UTILS}) -target_include_directories(PLEGMAutils SYSTEM PRIVATE ${QUDA_SRC}/lib ) +#cuda_add_library(PLEGMAutils STATIC ${PLEGMA_UTILS}) +add_library(PLEGMAutils STATIC ${PLEGMA_UTILS}) +if(${PLEGMA_TARGET_TYPE} STREQUAL "HIP") +target_include_directories(PLEGMAutils SYSTEM PRIVATE ${QUDA_SRC}/lib ${QUDA_HOME}/include/targets/hip) +endif() +if(${PLEGMA_TARGET_TYPE} STREQUAL "CUDA") +target_include_directories(PLEGMAutils SYSTEM PRIVATE ${QUDA_SRC}/lib ${QUDA_HOME}/include/targets/cuda) +endif() + add_dependencies(PLEGMAutils BoostPP)