Marconi code

CMakeLists.txt

@@ -82,6 +82,11 @@ if(GIT_FOUND)
       add_compile_definitions(QUDA_INCLUDES_COMMIT_1dec1db)
     endif()
   endif()
+else
+  add_compile_definitions(QUDA_INCLUDES_COMMIT_b08233a)
+  add_compile_definitions(QUDA_INCLUDES_COMMIT_775a033)	
+  add_compile_definitions(QUDA_INCLUDES_COMMIT_55782743)
+  add_compile_definitions(QUDA_INCLUDES_COMMIT_55782743)
 endif(GIT_FOUND)
 
 include(ExternalProject)
@@ -112,8 +117,8 @@ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS DEVEL RELEASE STRICT DEBUG
 string(REPLACE ":" ";" LIBRARY_DIRS $ENV{LD_LIBRARY_PATH})
 
 set(DEFAULT_GPU_ARCH sm_60)
-set(GPU_ARCH ${DEFAULT_GPU_ARCH} CACHE STRING "set the GPU architecture (sm_20, sm_21, sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_70)")
-set_property(CACHE GPU_ARCH PROPERTY STRINGS sm_20 sm_21 sm_30 sm_35 sm_37 sm_50 sm_52 sm_60 sm_70)
+set(GPU_ARCH ${DEFAULT_GPU_ARCH} CACHE STRING "set the GPU architecture (sm_20, sm_21, sm_30, sm_35, sm_37, sm_50, sm_52, sm_60, sm_70, sm_80)")
+set_property(CACHE GPU_ARCH PROPERTY STRINGS sm_20 sm_21 sm_30 sm_35 sm_37 sm_50 sm_52 sm_60 sm_70 sm_80)
 
 # max threads
 set(PLEGMA_MAX_THREADS 256 CACHE INT "maximum number of threads per block in tuner")
@@ -329,7 +334,7 @@ set(HDF5_PREFER_PARALLEL TRUE)
 find_package(HDF5)
 if(HDF5_FOUND)
   add_definitions(-DH5_HAVE_PARALLEL)
-  include_directories(SYSTEM ${HDF5_INCLUDE_DIRS})
+  include_directories(${HDF5_INCLUDE_DIRS})
   FIND_LIBRARY(PLEGMA_Z_LIB z ${LIBRARY_DIRS})	
 #link_libraries(${HDF5_C_LIBRARY_hdf5})
   message(STATUS "HDF5 version ${HDF5_VERSION} found, include dir: ${HDF5_INCLUDE_DIRS}, libraries ${HDF5_C_LIBRARY_hdf5}")

include/PLEGMA_Correlator.h

@@ -193,7 +193,15 @@ namespace plegma {
     }
     void contractMesons(PLEGMA_Propagator<Float> &prop1,
 			PLEGMA_Propagator<Float> &prop2);
-
+
+    void contractMesonsNew(PLEGMA_Propagator<Float> &prop1,
+			   PLEGMA_Propagator<Float> &prop2,
+			   bool all_cols=true);
+
+    void contractMesonsOpen(PLEGMA_Propagator<Float> &prop1,
+			   PLEGMA_Propagator<Float> &prop2,
+			   bool all_cols=true);
+
     void contractBaryons(PLEGMA_Propagator<Float> &prop1,
 			 PLEGMA_Propagator<Float> &prop2);
 

include/global/PLEGMA_structs.h

@@ -27,6 +27,27 @@ inline std::istream& operator >> (std::istream &i, site &x){
 }
 
 
+struct momentum : std::array<int,3> {
+  momentum() = default;
+  momentum(const std::array<int,3>& val) : std::array<int,3>(val) {}
+};
+inline std::ostream& operator << (std::ostream &o, momentum &x){
+  o<<x[0];
+  for(int i=1; i<3; i++)   o<<"-"<<x[i];
+  return o;
+}
+inline std::istream& operator >> (std::istream &i, momentum &x){
+  for(int j=0; j<3; j++) {
+    bool check = static_cast<bool> (i >> x[j]);
+    if(!check) {
+      PLEGMA_warning("Not enough arguments to unpack momentum\n");
+      break;
+    }
+  }
+  return i;
+}
+
+
 // Global variable for mom list
 struct tex_mom_list {
   size_t Nmoms;
@@ -111,6 +132,7 @@ struct global_vars {
   }
   void copyToDevice() {
     for(size_t i = 0; i != globals.size(); i++) {
+      PLEGMA_printf("%s\n",globals[i].get_value().c_str());
       globals[i].copyToDeviceConstant();
     }
   }

include/utils/PLEGMA_params.h

@@ -33,6 +33,7 @@ define(std::vector<std::string> listVecs);
 define(std::string pathListSourcePositions);
 define(std::string pathListMomenta);
 define(std::vector<site> sourcePositions, {});
+define(std::vector<momentum> momenta, {});
 define(int maxQsq, 64);
 define(FILE_FORMAT corr_file_format, HDF5_FORMAT);
 define(CORR_SPACE corr_space, MOMENTUM_SPACE);

include/utils/PLEGMA_readList.h

@@ -44,3 +44,20 @@ inline void readSourceList() {
   }
 }
 
+inline void readMomentaList() {
+  std::ifstream file(pathListMomenta.c_str(), std::ifstream::in);
+  if(file.fail()) PLEGMA_error("Cannot open file to read momenta list: %s\n", pathListMomenta.c_str());
+  momentum mom;
+  while (file >> mom) {
+    momenta.push_back(mom);
+  }
+  file.close();
+  PLEGMA_printf("Read %d momenta.\n", momenta.size());
+  if(verbosity) {
+    PLEGMA_printf("\nList of read momenta:\n");
+    for(int j=0; j<momenta.size(); j++) {
+      PLEGMA_printf("mom[%d]: %d-%d-%d\n", j, momenta[j][0], momenta[j][1], momenta[j][2]);
+    }
+  }
+}
+

include/utils/QUDA_params.h

@@ -8,9 +8,9 @@
 #endif
 
 #ifdef MULTI_GPU
-define(int device, -1);
+define(int device_number, -1);
 #else
-define(int device, 0);
+define(int device_number, 0);
 #endif
 
 define(bool qudaInitialized, false);

include/utils/QUDA_types.h

@@ -453,8 +453,6 @@ inline QudaEigSpectrumType get_eigensolution_type(std::string s) {
 inline  QudaEigType get_eigensolver(std::string s){
   if(s=="trlan" || s=="trlm")
     return QUDA_EIG_TR_LANCZOS;
-  else if(s=="irlan" || s=="irlm")
-    return QUDA_EIG_IR_LANCZOS;
   else if(s=="irarn" || s=="iram")
     return QUDA_EIG_IR_ARNOLDI;
   else {

lib/PLEGMA.cu

@@ -7,11 +7,13 @@
 #include <limits>
 #include <string.h>
 #include <PLEGMA_io.h>
+#include <communicator_quda.h>
+#include <comm_quda.h>
 
 //#define TIMING_REPORT
 using namespace plegma;
-extern Topology *default_topo;
 std::vector<std::string> HDF5::open_files;
+Communicator &get_current_communicator();
 
 void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){
   HGC_hold_exit = false;
@@ -25,7 +27,7 @@ void plegma::PLEGMA_init(int localL[4], int nProcs[4], int verbosity){
     for(int i = 0 ; i < N_DIMS ; i++)
       HGC_localL[i] = localL[i];
 
-    HGC_default_topo = default_topo;
+    HGC_default_topo = get_current_communicator().default_topo;
     HGC_verbosity = verbosity;
     for(int i = 0 ; i < N_DIMS ; i++) {
       HGC_nProc[i] = nProcs[i];

lib/PLEGMA_Correlator.cu

@@ -3,6 +3,8 @@
 #include <PLEGMA_Propagator.h>
 #include <string>
 #include <PLEGMA_mesons.cuh>
+#include <PLEGMA_mesonsNew.cuh>
+#include <PLEGMA_mesonsOpen.cuh>
 #include <PLEGMA_baryons.cuh>
 #include <PLEGMA_threep.cuh>
 #include <functional>
@@ -39,16 +41,47 @@ void PLEGMA_Correlator<Float>::
 contractMesons(PLEGMA_Propagator<Float> &prop1,
 	       PLEGMA_Propagator<Float> &prop2 ){
 
-  shape = {10};
+  shape = {16};
   datasets =  {"twop_meson_1", "twop_meson_2"};
   groups =  {"mesons"};
-  description = "pseudoscalar, scalar, g5g1, g5g2, g5g3, g5g4, g1, g2, g3, g4";
+  description = "pseudoscalar, scalar, g5g1, g5g2, g5g3, g5g4, g1, g2, g3, g4,s12,s13,s23,s41,s42,s43";
 
   initialize();
   contract_mesons(prop1,prop2,*this);
 }
 
 
+template<typename Float>
+void PLEGMA_Correlator<Float>::
+contractMesonsNew(PLEGMA_Propagator<Float> &prop1,
+		  PLEGMA_Propagator<Float> &prop2,
+		  bool all_cols){
+
+  shape = {16};
+  datasets =  {"twop_meson_new"};
+  groups =  {"mesons"};
+  description = "pseudoscalar, scalar, g5g1, g5g2, g5g3, g5g4, g1, g2, g3, g4,s12,s13,s23,s41,s42,s43";
+
+  initialize();
+  contract_mesons_new(prop1,prop2,*this,all_cols);
+}
+
+template<typename Float>
+void PLEGMA_Correlator<Float>::
+contractMesonsOpen(PLEGMA_Propagator<Float> &prop1,
+		  PLEGMA_Propagator<Float> &prop2,
+		  bool all_cols){
+
+  shape = {4,4,4,4};
+  datasets =  {"twop_meson_open"};
+  groups =  {"mesons"};
+  description = "open_indeces, prop1_mu,nu prop2_ku,lu";
+
+  initialize();
+  contract_mesons_open(prop1,prop2,*this,all_cols);
+}
+
+
 template<typename Float>
 void PLEGMA_Correlator<Float>::
 contractBaryons(PLEGMA_Propagator<Float> &prop1,

lib/PLEGMA_Vector.cu

@@ -10,7 +10,7 @@
 #include <PLEGMA_gammas.h>
 #include <kernels/PLEGMA_gammas_scatt.cuh>
 #endif
-
+#include <communicator_quda.h>
 using namespace plegma;
 using namespace quda;