[DO NOT MERGE] PTBC status

doc/sample-input/sample-ptbc-hmc-rgmixedcg-tmcloverdetratio.input

@@ -0,0 +1,137 @@
+# this is a sample input file for cloverdet + cloverdetratio using
+# DDalphaAMG as a solver
+
+L=8
+T=8
+Measurements = 20
+Startcondition = hot
+2KappaMu = 0.01
+CSW = 1.00
+kappa = 0.138
+NSave = 500000
+ThetaT = 1.0
+UseEvenOdd = yes
+ReversibilityCheck = yes
+ReversibilityCheckIntervall = 4
+InitialStoreCounter = 0
+DebugLevel = 1
+
+
+# complex PTBC topo example, 2 periodic, 2 tentacles
+# 2 periodic instances connected by dummy defect 2
+
+BeginPTBC
+
+  BeginPTBCDefect 0
+    Extents = 2, 2, 2, 2
+    Position = 1, 1, 1, 1
+  EndPTBCDefect
+
+  BeginPTBCDefect 1
+    Extents = 0, 2, 2, 2
+    Position = 4, 4, 4, 4
+  EndPTBCDefect
+
+  BeginPTBCDefect 2
+    Extents = 0, 0, 0, 1
+    Position = 0, 0, 3, 0
+  EndPTBCDefect
+
+  BeginPTBCDefect 3
+    Extents = 0, 1, 1, 1
+    Position = 6, 6, 6, 6
+  EndPTBCDefect
+
+  BeginPTBCInstance 0
+    Coefficients = 1.0, 1.0
+    Defects = 0, 2
+  EndPTBCInstance
+
+  BeginPTBCInstance 1
+    Coefficients = 0.0
+    Defects = 0
+  EndPTBCInstance
+
+  BeginPTBCInstance 2
+    Coefficients = 1.0, 1.0, 1.0
+    Defects = 1, 2, 3
+  EndPTBCInstance
+
+  BeginPTBCInstance 3
+    Coefficients = 0.1, 0.0
+    Defects = 1, 3
+  EndPTBCInstance
+
+EndPTBC
+
+
+
+
+# since this is a test file, we employ the reproducible random numbers mode
+ReproduceRandomNumbers = yes
+Seed = 127782
+
+BeginMeasurement CORRELATORS
+  Frequency = 2
+EndMeasurement
+
+BeginMonomial GAUGE
+  Type = Wilson
+  beta = 5.60
+  Timescale = 0
+EndMonomial
+
+BeginMonomial CLOVERDET
+  Timescale = 1
+  2KappaMu = 0.01
+  CSW = 1.00
+  # nominator shift
+  rho = 0.1
+  kappa = 0.138
+  AcceptancePrecision =  1.e-20
+  ForcePrecision = 1.e-14
+  Name = cloverdet
+  solver = rgmixedcg
+  usesloppyprecision = single
+EndMonomial
+
+BeginMonomial CLOVERDETRATIO
+  Timescale = 2
+  2KappaMu = 0.01
+  # nominator shift
+  rho = 0.0
+  # denominator shift, should match CLOVERDET shift
+  rho2 = 0.1
+  CSW = 1.00
+  kappa = 0.138
+  AcceptancePrecision =  1.e-20
+  ForcePrecision = 1.e-16
+  Name = cloverdetratio
+  solver = rgmixedcg
+  usesloppyprecision = single
+EndMonomial
+
+BeginIntegrator 
+  Type0 = 2MNFG
+  Type1 = 2MNFG
+  Type2 = 2MNFG
+  IntegrationSteps0 = 1
+  IntegrationSteps1 = 1
+  IntegrationSteps2 = 4
+  tau = 1.00
+  Lambda0 = 0.16666667
+  Lambda1 = 0.16666667
+  Lambda2 = 0.16666667
+  NumberOfTimescales = 3
+EndIntegrator
+
+BeginOperator CLOVER
+  2KappaMu = 0.01
+  CSW = 1.00
+  kappa = 0.138
+  SolverPrecision = 1e-16
+  MaxSolverIterations = 1000
+  useevenodd = yes
+  solver = rgmixedcg
+  usesloppyprecision = single
+EndOperator

src/bin/benchmark.c

@@ -102,7 +102,7 @@ int main(int argc, char *argv[]) {
 #else
   MPI_Init(&argc, &argv);
 #endif
-  MPI_Comm_rank(MPI_COMM_WORLD, &g_proc_id);
+  MPI_Comm_rank(app()->mpi.comm, &g_proc_id);
 
 #else
   g_proc_id = 0;
@@ -234,7 +234,7 @@ int main(int argc, char *argv[]) {
     antioptaway = 0.0;
     /* compute approximately how many applications we need to do to get a reliable measurement */
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
 #endif
     t1 = gettime();
     for (j = 0; j < j_max; j++) {
@@ -248,14 +248,14 @@ int main(int argc, char *argv[]) {
     // division by g_nproc because we will average over processes
     j = (int)(ceil(j_max * 31.0 / dt / g_nproc));
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&j, &j_max, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&j, &j_max, 1, MPI_INT, MPI_SUM, app()->mpi.comm);
 #else
     j_max = j;
 #endif
 
     /* perform the actual benchmark */
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
 #endif
     t1 = gettime();
     antioptaway = 0.0;
@@ -268,14 +268,14 @@ int main(int argc, char *argv[]) {
     }
     dt = gettime() - t1;
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
 #else
     sdt = dt;
 #endif
 
     qdt = dt * dt;
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&qdt, &sqdt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&qdt, &sqdt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
 #else
     sqdt = qdt;
 #endif
@@ -321,9 +321,9 @@ int main(int argc, char *argv[]) {
     dt2 = t2 - t1;
     /* compute the bandwidth */
     dt = dts - dt2;
-    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
     sdt = sdt / ((double)g_nproc);
-    MPI_Allreduce(&dt2, &dt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&dt2, &dt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
     dt = dt / ((double)g_nproc);
     dt = 1.0e6f * dt / ((double)(k_max * j_max * (VOLUME)));
     if (g_proc_id == 0) {
@@ -365,7 +365,7 @@ int main(int argc, char *argv[]) {
 
     /* estimate a reasonable number of applications to get a reliable measurement */
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
 #endif
     t1 = gettime();
     for (j = 0; j < j_max; j++) {
@@ -379,14 +379,14 @@ int main(int argc, char *argv[]) {
     // division by g_nproc because we will average over processes using  MPI_SUM
     j = (int)(ceil(j_max * 31.0 / dt / g_nproc));
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&j, &j_max, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&j, &j_max, 1, MPI_INT, MPI_SUM, app()->mpi.comm);
 #else
     j_max = j;
 #endif
 
     /* perform the actual measurement */
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
 #endif
     t1 = gettime();
     for (j = 0; j < j_max; j++) {
@@ -398,13 +398,13 @@ int main(int argc, char *argv[]) {
     t2 = gettime();
     dt = t2 - t1;
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&dt, &sdt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
 #else
     sdt = dt;
 #endif
     qdt = dt * dt;
 #ifdef TM_USE_MPI
-    MPI_Allreduce(&qdt, &sqdt, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    MPI_Allreduce(&qdt, &sqdt, 1, MPI_DOUBLE, MPI_SUM, app()->mpi.comm);
 #else
     sqdt = qdt;
 #endif
@@ -451,7 +451,7 @@ int main(int argc, char *argv[]) {
   free_spinor_field();
   free_moment_field();
 #ifdef TM_USE_MPI
-  MPI_Barrier(MPI_COMM_WORLD);
+  MPI_Barrier(app()->mpi.comm);
   MPI_Finalize();
 #endif
   return (0);

src/bin/deriv_mg_tune.c

@@ -357,7 +357,7 @@ int main(int argc, char *argv[]) {
   _endQuda();
 #endif
 #ifdef TM_USE_MPI
-  MPI_Barrier(MPI_COMM_WORLD);
+  MPI_Barrier(app()->mpi.comm);
   MPI_Finalize();
 #endif
 

src/bin/hmc_tm.c

@@ -73,6 +73,7 @@
 #ifdef TM_USE_QUDA
 #include "quda_interface.h"
 #endif
+#include "ptbc.h"
 
 extern int nstore;
 
@@ -142,6 +143,13 @@ int main(int argc, char *argv[]) {
   NO_OF_SPINORFIELDS_32 = 6;
 
   tmlqcd_mpi_init(argc, argv);
+
+  // initialise ptbc
+  if (app()->ptbc.active) {
+    init_ptbc_tree();
+    if (g_proc_id == 0) print_ptbc_topo();
+  }
+
   tm_stopwatch_push(&g_timers, "HMC", "");
 
   if (nstore == -1) {
@@ -479,7 +487,7 @@ int main(int argc, char *argv[]) {
 
           sleep(io_timeout);
 #ifdef TM_USE_MPI
-          MPI_Barrier(MPI_COMM_WORLD);
+          MPI_Barrier(app()->mpi.comm);
 #endif
         }
       /* Now move .conf.tmp into place */
@@ -528,7 +536,7 @@ int main(int argc, char *argv[]) {
     }
 
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
 #endif
     if (ix == 0 && g_proc_id == 0) {
       countfile = fopen("history_hmc_tm", "a");
@@ -581,7 +589,7 @@ int main(int argc, char *argv[]) {
   _endQuda();
 #endif
 #ifdef TM_USE_MPI
-  MPI_Barrier(MPI_COMM_WORLD);
+  MPI_Barrier(app()->mpi.comm);
   MPI_Finalize();
 #endif
 

src/bin/invert.c

@@ -448,7 +448,7 @@ int main(int argc, char *argv[]) {
   _endQuda();
 #endif
 #ifdef TM_USE_MPI
-  MPI_Barrier(MPI_COMM_WORLD);
+  MPI_Barrier(app()->mpi.comm);
   MPI_Finalize();
 #endif
   return (0);

src/bin/offline_measurement.c

@@ -294,7 +294,7 @@ int main(int argc, char *argv[]) {
   free(input_filename);
 
 #ifdef TM_USE_MPI
-  MPI_Barrier(MPI_COMM_WORLD);
+  MPI_Barrier(app()->mpi.comm);
   MPI_Finalize();
 #endif
   return (0);

src/bin/tests/hopping_test.c

@@ -318,7 +318,7 @@ int main(int argc, char *argv[]) {
     }
 
 #ifdef TM_USE_MPI
-    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(app()->mpi.comm);
     MPI_Finalize();
 #endif
   }